]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/nvme/host/core.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:


78dc6f624d52fe3ed0091a8fcf6dfb8772d932f4
[mirror_ubuntu-bionic-kernel.git] / drivers / nvme / host / core.c
1 /*
2 * NVM Express device driver
3 * Copyright (c) 2011-2014, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 */
14
15 #include <linux/blkdev.h>
16 #include <linux/blk-mq.h>
17 #include <linux/delay.h>
18 #include <linux/errno.h>
19 #include <linux/hdreg.h>
20 #include <linux/kernel.h>
21 #include <linux/module.h>
22 #include <linux/list_sort.h>
23 #include <linux/slab.h>
24 #include <linux/types.h>
25 #include <linux/pr.h>
26 #include <linux/ptrace.h>
27 #include <linux/nvme_ioctl.h>
28 #include <linux/t10-pi.h>
29 #include <linux/pm_qos.h>
30 #include <asm/unaligned.h>
31
32 #include "nvme.h"
33 #include "fabrics.h"
34
35 #define NVME_MINORS (1U << MINORBITS)
36
37 unsigned int admin_timeout = 60;
38 module_param(admin_timeout, uint, 0644);
39 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
40 EXPORT_SYMBOL_GPL(admin_timeout);
41
42 unsigned int nvme_io_timeout = 30;
43 module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
44 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
45 EXPORT_SYMBOL_GPL(nvme_io_timeout);
46
47 static unsigned char shutdown_timeout = 5;
48 module_param(shutdown_timeout, byte, 0644);
49 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
50
51 static u8 nvme_max_retries = 5;
52 module_param_named(max_retries, nvme_max_retries, byte, 0644);
53 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
54
55 static unsigned long default_ps_max_latency_us = 100000;
56 module_param(default_ps_max_latency_us, ulong, 0644);
57 MODULE_PARM_DESC(default_ps_max_latency_us,
58 "max power saving latency for new devices; use PM QOS to change per device");
59
60 static bool force_apst;
61 module_param(force_apst, bool, 0644);
62 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
63
64 static bool streams;
65 module_param(streams, bool, 0644);
66 MODULE_PARM_DESC(streams, "turn on support for Streams write directives");
67
68 struct workqueue_struct *nvme_wq;
69 EXPORT_SYMBOL_GPL(nvme_wq);
70
71 static DEFINE_IDA(nvme_subsystems_ida);
72 static LIST_HEAD(nvme_subsystems);
73 static DEFINE_MUTEX(nvme_subsystems_lock);
74
75 static DEFINE_IDA(nvme_instance_ida);
76 static dev_t nvme_chr_devt;
77 static struct class *nvme_class;
78 static struct class *nvme_subsys_class;
79
80 static void nvme_ns_remove(struct nvme_ns *ns);
81 static int nvme_revalidate_disk(struct gendisk *disk);
82
83 static __le32 nvme_get_log_dw10(u8 lid, size_t size)
84 {
85 return cpu_to_le32((((size / 4) - 1) << 16) | lid);
86 }
87
88 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
89 {
90 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
91 return -EBUSY;
92 if (!queue_work(nvme_wq, &ctrl->reset_work))
93 return -EBUSY;
94 return 0;
95 }
96 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
97
98 static int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
99 {
100 int ret;
101
102 ret = nvme_reset_ctrl(ctrl);
103 if (!ret)
104 flush_work(&ctrl->reset_work);
105 return ret;
106 }
107
108 static void nvme_delete_ctrl_work(struct work_struct *work)
109 {
110 struct nvme_ctrl *ctrl =
111 container_of(work, struct nvme_ctrl, delete_work);
112
113 flush_work(&ctrl->reset_work);
114 nvme_stop_ctrl(ctrl);
115 nvme_remove_namespaces(ctrl);
116 ctrl->ops->delete_ctrl(ctrl);
117 nvme_uninit_ctrl(ctrl);
118 nvme_put_ctrl(ctrl);
119 }
120
121 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
122 {
123 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
124 return -EBUSY;
125 if (!queue_work(nvme_wq, &ctrl->delete_work))
126 return -EBUSY;
127 return 0;
128 }
129 EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
130
131 int nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
132 {
133 int ret = 0;
134
135 /*
136 * Keep a reference until the work is flushed since ->delete_ctrl
137 * can free the controller.
138 */
139 nvme_get_ctrl(ctrl);
140 ret = nvme_delete_ctrl(ctrl);
141 if (!ret)
142 flush_work(&ctrl->delete_work);
143 nvme_put_ctrl(ctrl);
144 return ret;
145 }
146 EXPORT_SYMBOL_GPL(nvme_delete_ctrl_sync);
147
148 static inline bool nvme_ns_has_pi(struct nvme_ns *ns)
149 {
150 return ns->pi_type && ns->ms == sizeof(struct t10_pi_tuple);
151 }
152
153 static blk_status_t nvme_error_status(struct request *req)
154 {
155 switch (nvme_req(req)->status & 0x7ff) {
156 case NVME_SC_SUCCESS:
157 return BLK_STS_OK;
158 case NVME_SC_CAP_EXCEEDED:
159 return BLK_STS_NOSPC;
160 case NVME_SC_ONCS_NOT_SUPPORTED:
161 return BLK_STS_NOTSUPP;
162 case NVME_SC_WRITE_FAULT:
163 case NVME_SC_READ_ERROR:
164 case NVME_SC_UNWRITTEN_BLOCK:
165 case NVME_SC_ACCESS_DENIED:
166 case NVME_SC_READ_ONLY:
167 return BLK_STS_MEDIUM;
168 case NVME_SC_GUARD_CHECK:
169 case NVME_SC_APPTAG_CHECK:
170 case NVME_SC_REFTAG_CHECK:
171 case NVME_SC_INVALID_PI:
172 return BLK_STS_PROTECTION;
173 case NVME_SC_RESERVATION_CONFLICT:
174 return BLK_STS_NEXUS;
175 default:
176 return BLK_STS_IOERR;
177 }
178 }
179
180 static inline bool nvme_req_needs_retry(struct request *req)
181 {
182 if (blk_noretry_request(req))
183 return false;
184 if (nvme_req(req)->status & NVME_SC_DNR)
185 return false;
186 if (nvme_req(req)->retries >= nvme_max_retries)
187 return false;
188 if (blk_queue_dying(req->q))
189 return false;
190 return true;
191 }
192
193 void nvme_complete_rq(struct request *req)
194 {
195 if (unlikely(nvme_req(req)->status && nvme_req_needs_retry(req))) {
196 nvme_req(req)->retries++;
197 blk_mq_requeue_request(req, true);
198 return;
199 }
200
201 blk_mq_end_request(req, nvme_error_status(req));
202 }
203 EXPORT_SYMBOL_GPL(nvme_complete_rq);
204
205 void nvme_cancel_request(struct request *req, void *data, bool reserved)
206 {
207 if (!blk_mq_request_started(req))
208 return;
209
210 dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
211 "Cancelling I/O %d", req->tag);
212
213 nvme_req(req)->status = NVME_SC_ABORT_REQ;
214 blk_mq_complete_request(req);
215
216 }
217 EXPORT_SYMBOL_GPL(nvme_cancel_request);
218
219 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
220 enum nvme_ctrl_state new_state)
221 {
222 enum nvme_ctrl_state old_state;
223 unsigned long flags;
224 bool changed = false;
225
226 spin_lock_irqsave(&ctrl->lock, flags);
227
228 old_state = ctrl->state;
229 switch (new_state) {
230 case NVME_CTRL_LIVE:
231 switch (old_state) {
232 case NVME_CTRL_NEW:
233 case NVME_CTRL_RESETTING:
234 case NVME_CTRL_RECONNECTING:
235 changed = true;
236 /* FALLTHRU */
237 default:
238 break;
239 }
240 break;
241 case NVME_CTRL_RESETTING:
242 switch (old_state) {
243 case NVME_CTRL_NEW:
244 case NVME_CTRL_LIVE:
245 changed = true;
246 /* FALLTHRU */
247 default:
248 break;
249 }
250 break;
251 case NVME_CTRL_RECONNECTING:
252 switch (old_state) {
253 case NVME_CTRL_LIVE:
254 case NVME_CTRL_RESETTING:
255 changed = true;
256 /* FALLTHRU */
257 default:
258 break;
259 }
260 break;
261 case NVME_CTRL_DELETING:
262 switch (old_state) {
263 case NVME_CTRL_LIVE:
264 case NVME_CTRL_RESETTING:
265 case NVME_CTRL_RECONNECTING:
266 changed = true;
267 /* FALLTHRU */
268 default:
269 break;
270 }
271 break;
272 case NVME_CTRL_DEAD:
273 switch (old_state) {
274 case NVME_CTRL_DELETING:
275 changed = true;
276 /* FALLTHRU */
277 default:
278 break;
279 }
280 break;
281 default:
282 break;
283 }
284
285 if (changed)
286 ctrl->state = new_state;
287
288 spin_unlock_irqrestore(&ctrl->lock, flags);
289
290 return changed;
291 }
292 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
293
294 static void nvme_free_ns(struct kref *kref)
295 {
296 struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
297
298 if (ns->ndev)
299 nvme_nvm_unregister(ns);
300
301 put_disk(ns->disk);
302 ida_simple_remove(&ns->ctrl->ns_ida, ns->instance);
303 nvme_put_ctrl(ns->ctrl);
304 kfree(ns);
305 }
306
307 static void nvme_put_ns(struct nvme_ns *ns)
308 {
309 kref_put(&ns->kref, nvme_free_ns);
310 }
311
312 struct request *nvme_alloc_request(struct request_queue *q,
313 struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid)
314 {
315 unsigned op = nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
316 struct request *req;
317
318 if (qid == NVME_QID_ANY) {
319 req = blk_mq_alloc_request(q, op, flags);
320 } else {
321 req = blk_mq_alloc_request_hctx(q, op, flags,
322 qid ? qid - 1 : 0);
323 }
324 if (IS_ERR(req))
325 return req;
326
327 req->cmd_flags |= REQ_FAILFAST_DRIVER;
328 nvme_req(req)->cmd = cmd;
329
330 return req;
331 }
332 EXPORT_SYMBOL_GPL(nvme_alloc_request);
333
334 static int nvme_toggle_streams(struct nvme_ctrl *ctrl, bool enable)
335 {
336 struct nvme_command c;
337
338 memset(&c, 0, sizeof(c));
339
340 c.directive.opcode = nvme_admin_directive_send;
341 c.directive.nsid = cpu_to_le32(NVME_NSID_ALL);
342 c.directive.doper = NVME_DIR_SND_ID_OP_ENABLE;
343 c.directive.dtype = NVME_DIR_IDENTIFY;
344 c.directive.tdtype = NVME_DIR_STREAMS;
345 c.directive.endir = enable ? NVME_DIR_ENDIR : 0;
346
347 return nvme_submit_sync_cmd(ctrl->admin_q, &c, NULL, 0);
348 }
349
350 static int nvme_disable_streams(struct nvme_ctrl *ctrl)
351 {
352 return nvme_toggle_streams(ctrl, false);
353 }
354
355 static int nvme_enable_streams(struct nvme_ctrl *ctrl)
356 {
357 return nvme_toggle_streams(ctrl, true);
358 }
359
360 static int nvme_get_stream_params(struct nvme_ctrl *ctrl,
361 struct streams_directive_params *s, u32 nsid)
362 {
363 struct nvme_command c;
364
365 memset(&c, 0, sizeof(c));
366 memset(s, 0, sizeof(*s));
367
368 c.directive.opcode = nvme_admin_directive_recv;
369 c.directive.nsid = cpu_to_le32(nsid);
370 c.directive.numd = cpu_to_le32((sizeof(*s) >> 2) - 1);
371 c.directive.doper = NVME_DIR_RCV_ST_OP_PARAM;
372 c.directive.dtype = NVME_DIR_STREAMS;
373
374 return nvme_submit_sync_cmd(ctrl->admin_q, &c, s, sizeof(*s));
375 }
376
377 static int nvme_configure_directives(struct nvme_ctrl *ctrl)
378 {
379 struct streams_directive_params s;
380 int ret;
381
382 if (!(ctrl->oacs & NVME_CTRL_OACS_DIRECTIVES))
383 return 0;
384 if (!streams)
385 return 0;
386
387 ret = nvme_enable_streams(ctrl);
388 if (ret)
389 return ret;
390
391 ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL);
392 if (ret)
393 return ret;
394
395 ctrl->nssa = le16_to_cpu(s.nssa);
396 if (ctrl->nssa < BLK_MAX_WRITE_HINTS - 1) {
397 dev_info(ctrl->device, "too few streams (%u) available\n",
398 ctrl->nssa);
399 nvme_disable_streams(ctrl);
400 return 0;
401 }
402
403 ctrl->nr_streams = min_t(unsigned, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1);
404 dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams);
405 return 0;
406 }
407
408 /*
409 * Check if 'req' has a write hint associated with it. If it does, assign
410 * a valid namespace stream to the write.
411 */
412 static void nvme_assign_write_stream(struct nvme_ctrl *ctrl,
413 struct request *req, u16 *control,
414 u32 *dsmgmt)
415 {
416 enum rw_hint streamid = req->write_hint;
417
418 if (streamid == WRITE_LIFE_NOT_SET || streamid == WRITE_LIFE_NONE)
419 streamid = 0;
420 else {
421 streamid--;
422 if (WARN_ON_ONCE(streamid > ctrl->nr_streams))
423 return;
424
425 *control |= NVME_RW_DTYPE_STREAMS;
426 *dsmgmt |= streamid << 16;
427 }
428
429 if (streamid < ARRAY_SIZE(req->q->write_hints))
430 req->q->write_hints[streamid] += blk_rq_bytes(req) >> 9;
431 }
432
433 static inline void nvme_setup_flush(struct nvme_ns *ns,
434 struct nvme_command *cmnd)
435 {
436 memset(cmnd, 0, sizeof(*cmnd));
437 cmnd->common.opcode = nvme_cmd_flush;
438 cmnd->common.nsid = cpu_to_le32(ns->ns_id);
439 }
440
441 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
442 struct nvme_command *cmnd)
443 {
444 unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
445 struct nvme_dsm_range *range;
446 struct bio *bio;
447
448 range = kmalloc_array(segments, sizeof(*range), GFP_ATOMIC);
449 if (!range)
450 return BLK_STS_RESOURCE;
451
452 __rq_for_each_bio(bio, req) {
453 u64 slba = nvme_block_nr(ns, bio->bi_iter.bi_sector);
454 u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
455
456 range[n].cattr = cpu_to_le32(0);
457 range[n].nlb = cpu_to_le32(nlb);
458 range[n].slba = cpu_to_le64(slba);
459 n++;
460 }
461
462 if (WARN_ON_ONCE(n != segments)) {
463 kfree(range);
464 return BLK_STS_IOERR;
465 }
466
467 memset(cmnd, 0, sizeof(*cmnd));
468 cmnd->dsm.opcode = nvme_cmd_dsm;
469 cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
470 cmnd->dsm.nr = cpu_to_le32(segments - 1);
471 cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
472
473 req->special_vec.bv_page = virt_to_page(range);
474 req->special_vec.bv_offset = offset_in_page(range);
475 req->special_vec.bv_len = sizeof(*range) * segments;
476 req->rq_flags |= RQF_SPECIAL_PAYLOAD;
477
478 return BLK_STS_OK;
479 }
480
481 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
482 struct request *req, struct nvme_command *cmnd)
483 {
484 struct nvme_ctrl *ctrl = ns->ctrl;
485 u16 control = 0;
486 u32 dsmgmt = 0;
487
488 if (req->cmd_flags & REQ_FUA)
489 control |= NVME_RW_FUA;
490 if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
491 control |= NVME_RW_LR;
492
493 if (req->cmd_flags & REQ_RAHEAD)
494 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
495
496 memset(cmnd, 0, sizeof(*cmnd));
497 cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
498 cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
499 cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
500 cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
501
502 if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams)
503 nvme_assign_write_stream(ctrl, req, &control, &dsmgmt);
504
505 if (ns->ms) {
506 /*
507 * If formated with metadata, the block layer always provides a
508 * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled. Else
509 * we enable the PRACT bit for protection information or set the
510 * namespace capacity to zero to prevent any I/O.
511 */
512 if (!blk_integrity_rq(req)) {
513 if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
514 return BLK_STS_NOTSUPP;
515 control |= NVME_RW_PRINFO_PRACT;
516 }
517
518 switch (ns->pi_type) {
519 case NVME_NS_DPS_PI_TYPE3:
520 control |= NVME_RW_PRINFO_PRCHK_GUARD;
521 break;
522 case NVME_NS_DPS_PI_TYPE1:
523 case NVME_NS_DPS_PI_TYPE2:
524 control |= NVME_RW_PRINFO_PRCHK_GUARD |
525 NVME_RW_PRINFO_PRCHK_REF;
526 cmnd->rw.reftag = cpu_to_le32(
527 nvme_block_nr(ns, blk_rq_pos(req)));
528 break;
529 }
530 }
531
532 cmnd->rw.control = cpu_to_le16(control);
533 cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
534 return 0;
535 }
536
537 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
538 struct nvme_command *cmd)
539 {
540 blk_status_t ret = BLK_STS_OK;
541
542 if (!(req->rq_flags & RQF_DONTPREP)) {
543 nvme_req(req)->retries = 0;
544 nvme_req(req)->flags = 0;
545 req->rq_flags |= RQF_DONTPREP;
546 }
547
548 switch (req_op(req)) {
549 case REQ_OP_DRV_IN:
550 case REQ_OP_DRV_OUT:
551 memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
552 break;
553 case REQ_OP_FLUSH:
554 nvme_setup_flush(ns, cmd);
555 break;
556 case REQ_OP_WRITE_ZEROES:
557 /* currently only aliased to deallocate for a few ctrls: */
558 case REQ_OP_DISCARD:
559 ret = nvme_setup_discard(ns, req, cmd);
560 break;
561 case REQ_OP_READ:
562 case REQ_OP_WRITE:
563 ret = nvme_setup_rw(ns, req, cmd);
564 break;
565 default:
566 WARN_ON_ONCE(1);
567 return BLK_STS_IOERR;
568 }
569
570 cmd->common.command_id = req->tag;
571 return ret;
572 }
573 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
574
575 /*
576 * Returns 0 on success. If the result is negative, it's a Linux error code;
577 * if the result is positive, it's an NVM Express status code
578 */
579 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
580 union nvme_result *result, void *buffer, unsigned bufflen,
581 unsigned timeout, int qid, int at_head,
582 blk_mq_req_flags_t flags)
583 {
584 struct request *req;
585 int ret;
586
587 req = nvme_alloc_request(q, cmd, flags, qid);
588 if (IS_ERR(req))
589 return PTR_ERR(req);
590
591 req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
592
593 if (buffer && bufflen) {
594 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
595 if (ret)
596 goto out;
597 }
598
599 blk_execute_rq(req->q, NULL, req, at_head);
600 if (result)
601 *result = nvme_req(req)->result;
602 if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
603 ret = -EINTR;
604 else
605 ret = nvme_req(req)->status;
606 out:
607 blk_mq_free_request(req);
608 return ret;
609 }
610 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
611
612 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
613 void *buffer, unsigned bufflen)
614 {
615 return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
616 NVME_QID_ANY, 0, 0);
617 }
618 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
619
620 static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf,
621 unsigned len, u32 seed, bool write)
622 {
623 struct bio_integrity_payload *bip;
624 int ret = -ENOMEM;
625 void *buf;
626
627 buf = kmalloc(len, GFP_KERNEL);
628 if (!buf)
629 goto out;
630
631 ret = -EFAULT;
632 if (write && copy_from_user(buf, ubuf, len))
633 goto out_free_meta;
634
635 bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
636 if (IS_ERR(bip)) {
637 ret = PTR_ERR(bip);
638 goto out_free_meta;
639 }
640
641 bip->bip_iter.bi_size = len;
642 bip->bip_iter.bi_sector = seed;
643 ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
644 offset_in_page(buf));
645 if (ret == len)
646 return buf;
647 ret = -ENOMEM;
648 out_free_meta:
649 kfree(buf);
650 out:
651 return ERR_PTR(ret);
652 }
653
654 static int nvme_submit_user_cmd(struct request_queue *q,
655 struct nvme_command *cmd, void __user *ubuffer,
656 unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
657 u32 meta_seed, u32 *result, unsigned timeout)
658 {
659 bool write = nvme_is_write(cmd);
660 struct nvme_ns *ns = q->queuedata;
661 struct gendisk *disk = ns ? ns->disk : NULL;
662 struct request *req;
663 struct bio *bio = NULL;
664 void *meta = NULL;
665 int ret;
666
667 req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY);
668 if (IS_ERR(req))
669 return PTR_ERR(req);
670
671 req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
672
673 if (ubuffer && bufflen) {
674 ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
675 GFP_KERNEL);
676 if (ret)
677 goto out;
678 bio = req->bio;
679 bio->bi_disk = disk;
680 if (disk && meta_buffer && meta_len) {
681 meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,
682 meta_seed, write);
683 if (IS_ERR(meta)) {
684 ret = PTR_ERR(meta);
685 goto out_unmap;
686 }
687 }
688 }
689
690 blk_execute_rq(req->q, disk, req, 0);
691 if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
692 ret = -EINTR;
693 else
694 ret = nvme_req(req)->status;
695 if (result)
696 *result = le32_to_cpu(nvme_req(req)->result.u32);
697 if (meta && !ret && !write) {
698 if (copy_to_user(meta_buffer, meta, meta_len))
699 ret = -EFAULT;
700 }
701 kfree(meta);
702 out_unmap:
703 if (bio)
704 blk_rq_unmap_user(bio);
705 out:
706 blk_mq_free_request(req);
707 return ret;
708 }
709
710 static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status)
711 {
712 struct nvme_ctrl *ctrl = rq->end_io_data;
713
714 blk_mq_free_request(rq);
715
716 if (status) {
717 dev_err(ctrl->device,
718 "failed nvme_keep_alive_end_io error=%d\n",
719 status);
720 return;
721 }
722
723 schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
724 }
725
726 static int nvme_keep_alive(struct nvme_ctrl *ctrl)
727 {
728 struct nvme_command c;
729 struct request *rq;
730
731 memset(&c, 0, sizeof(c));
732 c.common.opcode = nvme_admin_keep_alive;
733
734 rq = nvme_alloc_request(ctrl->admin_q, &c, BLK_MQ_REQ_RESERVED,
735 NVME_QID_ANY);
736 if (IS_ERR(rq))
737 return PTR_ERR(rq);
738
739 rq->timeout = ctrl->kato * HZ;
740 rq->end_io_data = ctrl;
741
742 blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);
743
744 return 0;
745 }
746
747 static void nvme_keep_alive_work(struct work_struct *work)
748 {
749 struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
750 struct nvme_ctrl, ka_work);
751
752 if (nvme_keep_alive(ctrl)) {
753 /* allocation failure, reset the controller */
754 dev_err(ctrl->device, "keep-alive failed\n");
755 nvme_reset_ctrl(ctrl);
756 return;
757 }
758 }
759
760 void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
761 {
762 if (unlikely(ctrl->kato == 0))
763 return;
764
765 INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
766 schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
767 }
768 EXPORT_SYMBOL_GPL(nvme_start_keep_alive);
769
770 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
771 {
772 if (unlikely(ctrl->kato == 0))
773 return;
774
775 cancel_delayed_work_sync(&ctrl->ka_work);
776 }
777 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
778
779 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
780 {
781 struct nvme_command c = { };
782 int error;
783
784 /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
785 c.identify.opcode = nvme_admin_identify;
786 c.identify.cns = NVME_ID_CNS_CTRL;
787
788 *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
789 if (!*id)
790 return -ENOMEM;
791
792 error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
793 sizeof(struct nvme_id_ctrl));
794 if (error)
795 kfree(*id);
796 return error;
797 }
798
799 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid,
800 u8 *eui64, u8 *nguid, uuid_t *uuid)
801 {
802 struct nvme_command c = { };
803 int status;
804 void *data;
805 int pos;
806 int len;
807
808 c.identify.opcode = nvme_admin_identify;
809 c.identify.nsid = cpu_to_le32(nsid);
810 c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
811
812 data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
813 if (!data)
814 return -ENOMEM;
815
816 status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
817 NVME_IDENTIFY_DATA_SIZE);
818 if (status)
819 goto free_data;
820
821 for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
822 struct nvme_ns_id_desc *cur = data + pos;
823
824 if (cur->nidl == 0)
825 break;
826
827 switch (cur->nidt) {
828 case NVME_NIDT_EUI64:
829 if (cur->nidl != NVME_NIDT_EUI64_LEN) {
830 dev_warn(ctrl->device,
831 "ctrl returned bogus length: %d for NVME_NIDT_EUI64\n",
832 cur->nidl);
833 goto free_data;
834 }
835 len = NVME_NIDT_EUI64_LEN;
836 memcpy(eui64, data + pos + sizeof(*cur), len);
837 break;
838 case NVME_NIDT_NGUID:
839 if (cur->nidl != NVME_NIDT_NGUID_LEN) {
840 dev_warn(ctrl->device,
841 "ctrl returned bogus length: %d for NVME_NIDT_NGUID\n",
842 cur->nidl);
843 goto free_data;
844 }
845 len = NVME_NIDT_NGUID_LEN;
846 memcpy(nguid, data + pos + sizeof(*cur), len);
847 break;
848 case NVME_NIDT_UUID:
849 if (cur->nidl != NVME_NIDT_UUID_LEN) {
850 dev_warn(ctrl->device,
851 "ctrl returned bogus length: %d for NVME_NIDT_UUID\n",
852 cur->nidl);
853 goto free_data;
854 }
855 len = NVME_NIDT_UUID_LEN;
856 uuid_copy(uuid, data + pos + sizeof(*cur));
857 break;
858 default:
859 /* Skip unnkown types */
860 len = cur->nidl;
861 break;
862 }
863
864 len += sizeof(*cur);
865 }
866 free_data:
867 kfree(data);
868 return status;
869 }
870
871 static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *ns_list)
872 {
873 struct nvme_command c = { };
874
875 c.identify.opcode = nvme_admin_identify;
876 c.identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST;
877 c.identify.nsid = cpu_to_le32(nsid);
878 return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list, 0x1000);
879 }
880
881 static struct nvme_id_ns *nvme_identify_ns(struct nvme_ctrl *ctrl,
882 unsigned nsid)
883 {
884 struct nvme_id_ns *id;
885 struct nvme_command c = { };
886 int error;
887
888 /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
889 c.identify.opcode = nvme_admin_identify;
890 c.identify.nsid = cpu_to_le32(nsid);
891 c.identify.cns = NVME_ID_CNS_NS;
892
893 id = kmalloc(sizeof(*id), GFP_KERNEL);
894 if (!id)
895 return NULL;
896
897 error = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
898 if (error) {
899 dev_warn(ctrl->device, "Identify namespace failed\n");
900 kfree(id);
901 return NULL;
902 }
903
904 return id;
905 }
906
907 static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
908 void *buffer, size_t buflen, u32 *result)
909 {
910 struct nvme_command c;
911 union nvme_result res;
912 int ret;
913
914 memset(&c, 0, sizeof(c));
915 c.features.opcode = nvme_admin_set_features;
916 c.features.fid = cpu_to_le32(fid);
917 c.features.dword11 = cpu_to_le32(dword11);
918
919 ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
920 buffer, buflen, 0, NVME_QID_ANY, 0, 0);
921 if (ret >= 0 && result)
922 *result = le32_to_cpu(res.u32);
923 return ret;
924 }
925
926 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
927 {
928 u32 q_count = (*count - 1) | ((*count - 1) << 16);
929 u32 result;
930 int status, nr_io_queues;
931
932 status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
933 &result);
934 if (status < 0)
935 return status;
936
937 /*
938 * Degraded controllers might return an error when setting the queue
939 * count. We still want to be able to bring them online and offer
940 * access to the admin queue, as that might be only way to fix them up.
941 */
942 if (status > 0) {
943 dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
944 *count = 0;
945 } else {
946 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
947 *count = min(*count, nr_io_queues);
948 }
949
950 return 0;
951 }
952 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
953
954 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
955 {
956 struct nvme_user_io io;
957 struct nvme_command c;
958 unsigned length, meta_len;
959 void __user *metadata;
960
961 if (copy_from_user(&io, uio, sizeof(io)))
962 return -EFAULT;
963 if (io.flags)
964 return -EINVAL;
965
966 switch (io.opcode) {
967 case nvme_cmd_write:
968 case nvme_cmd_read:
969 case nvme_cmd_compare:
970 break;
971 default:
972 return -EINVAL;
973 }
974
975 length = (io.nblocks + 1) << ns->lba_shift;
976 meta_len = (io.nblocks + 1) * ns->ms;
977 metadata = (void __user *)(uintptr_t)io.metadata;
978
979 if (ns->ext) {
980 length += meta_len;
981 meta_len = 0;
982 } else if (meta_len) {
983 if ((io.metadata & 3) || !io.metadata)
984 return -EINVAL;
985 }
986
987 memset(&c, 0, sizeof(c));
988 c.rw.opcode = io.opcode;
989 c.rw.flags = io.flags;
990 c.rw.nsid = cpu_to_le32(ns->ns_id);
991 c.rw.slba = cpu_to_le64(io.slba);
992 c.rw.length = cpu_to_le16(io.nblocks);
993 c.rw.control = cpu_to_le16(io.control);
994 c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
995 c.rw.reftag = cpu_to_le32(io.reftag);
996 c.rw.apptag = cpu_to_le16(io.apptag);
997 c.rw.appmask = cpu_to_le16(io.appmask);
998
999 return nvme_submit_user_cmd(ns->queue, &c,
1000 (void __user *)(uintptr_t)io.addr, length,
1001 metadata, meta_len, io.slba, NULL, 0);
1002 }
1003
1004 static u32 nvme_known_admin_effects(u8 opcode)
1005 {
1006 switch (opcode) {
1007 case nvme_admin_format_nvm:
1008 return NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC |
1009 NVME_CMD_EFFECTS_CSE_MASK;
1010 case nvme_admin_sanitize_nvm:
1011 return NVME_CMD_EFFECTS_CSE_MASK;
1012 default:
1013 break;
1014 }
1015 return 0;
1016 }
1017
1018 static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1019 u8 opcode)
1020 {
1021 u32 effects = 0;
1022
1023 if (ns) {
1024 if (ctrl->effects)
1025 effects = le32_to_cpu(ctrl->effects->iocs[opcode]);
1026 if (effects & ~NVME_CMD_EFFECTS_CSUPP)
1027 dev_warn(ctrl->device,
1028 "IO command:%02x has unhandled effects:%08x\n",
1029 opcode, effects);
1030 return 0;
1031 }
1032
1033 if (ctrl->effects)
1034 effects = le32_to_cpu(ctrl->effects->iocs[opcode]);
1035 else
1036 effects = nvme_known_admin_effects(opcode);
1037
1038 /*
1039 * For simplicity, IO to all namespaces is quiesced even if the command
1040 * effects say only one namespace is affected.
1041 */
1042 if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
1043 nvme_start_freeze(ctrl);
1044 nvme_wait_freeze(ctrl);
1045 }
1046 return effects;
1047 }
1048
1049 static void nvme_update_formats(struct nvme_ctrl *ctrl)
1050 {
1051 struct nvme_ns *ns;
1052
1053 mutex_lock(&ctrl->namespaces_mutex);
1054 list_for_each_entry(ns, &ctrl->namespaces, list) {
1055 if (ns->disk && nvme_revalidate_disk(ns->disk))
1056 nvme_ns_remove(ns);
1057 }
1058 mutex_unlock(&ctrl->namespaces_mutex);
1059 }
1060
1061 static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
1062 {
1063 /*
1064 * Revalidate LBA changes prior to unfreezing. This is necessary to
1065 * prevent memory corruption if a logical block size was changed by
1066 * this command.
1067 */
1068 if (effects & NVME_CMD_EFFECTS_LBCC)
1069 nvme_update_formats(ctrl);
1070 if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK))
1071 nvme_unfreeze(ctrl);
1072 if (effects & NVME_CMD_EFFECTS_CCC)
1073 nvme_init_identify(ctrl);
1074 if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC))
1075 nvme_queue_scan(ctrl);
1076 }
1077
1078 static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1079 struct nvme_passthru_cmd __user *ucmd)
1080 {
1081 struct nvme_passthru_cmd cmd;
1082 struct nvme_command c;
1083 unsigned timeout = 0;
1084 u32 effects;
1085 int status;
1086
1087 if (!capable(CAP_SYS_ADMIN))
1088 return -EACCES;
1089 if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1090 return -EFAULT;
1091 if (cmd.flags)
1092 return -EINVAL;
1093
1094 memset(&c, 0, sizeof(c));
1095 c.common.opcode = cmd.opcode;
1096 c.common.flags = cmd.flags;
1097 c.common.nsid = cpu_to_le32(cmd.nsid);
1098 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1099 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1100 c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
1101 c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
1102 c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
1103 c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
1104 c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
1105 c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
1106
1107 if (cmd.timeout_ms)
1108 timeout = msecs_to_jiffies(cmd.timeout_ms);
1109
1110 effects = nvme_passthru_start(ctrl, ns, cmd.opcode);
1111 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1112 (void __user *)(uintptr_t)cmd.addr, cmd.data_len,
1113 (void __user *)(uintptr_t)cmd.metadata, cmd.metadata,
1114 0, &cmd.result, timeout);
1115 nvme_passthru_end(ctrl, effects);
1116
1117 if (status >= 0) {
1118 if (put_user(cmd.result, &ucmd->result))
1119 return -EFAULT;
1120 }
1121
1122 return status;
1123 }
1124
1125 static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
1126 unsigned int cmd, unsigned long arg)
1127 {
1128 struct nvme_ns *ns = bdev->bd_disk->private_data;
1129
1130 switch (cmd) {
1131 case NVME_IOCTL_ID:
1132 force_successful_syscall_return();
1133 return ns->ns_id;
1134 case NVME_IOCTL_ADMIN_CMD:
1135 return nvme_user_cmd(ns->ctrl, NULL, (void __user *)arg);
1136 case NVME_IOCTL_IO_CMD:
1137 return nvme_user_cmd(ns->ctrl, ns, (void __user *)arg);
1138 case NVME_IOCTL_SUBMIT_IO:
1139 return nvme_submit_io(ns, (void __user *)arg);
1140 default:
1141 #ifdef CONFIG_NVM
1142 if (ns->ndev)
1143 return nvme_nvm_ioctl(ns, cmd, arg);
1144 #endif
1145 if (is_sed_ioctl(cmd))
1146 return sed_ioctl(ns->ctrl->opal_dev, cmd,
1147 (void __user *) arg);
1148 return -ENOTTY;
1149 }
1150 }
1151
1152 static int nvme_open(struct block_device *bdev, fmode_t mode)
1153 {
1154 struct nvme_ns *ns = bdev->bd_disk->private_data;
1155
1156 if (!kref_get_unless_zero(&ns->kref))
1157 return -ENXIO;
1158 return 0;
1159 }
1160
1161 static void nvme_release(struct gendisk *disk, fmode_t mode)
1162 {
1163 nvme_put_ns(disk->private_data);
1164 }
1165
1166 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1167 {
1168 /* some standard values */
1169 geo->heads = 1 << 6;
1170 geo->sectors = 1 << 5;
1171 geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1172 return 0;
1173 }
1174
1175 #ifdef CONFIG_BLK_DEV_INTEGRITY
1176 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type)
1177 {
1178 struct blk_integrity integrity;
1179
1180 memset(&integrity, 0, sizeof(integrity));
1181 switch (pi_type) {
1182 case NVME_NS_DPS_PI_TYPE3:
1183 integrity.profile = &t10_pi_type3_crc;
1184 integrity.tag_size = sizeof(u16) + sizeof(u32);
1185 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1186 break;
1187 case NVME_NS_DPS_PI_TYPE1:
1188 case NVME_NS_DPS_PI_TYPE2:
1189 integrity.profile = &t10_pi_type1_crc;
1190 integrity.tag_size = sizeof(u16);
1191 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1192 break;
1193 default:
1194 integrity.profile = NULL;
1195 break;
1196 }
1197 integrity.tuple_size = ms;
1198 blk_integrity_register(disk, &integrity);
1199 blk_queue_max_integrity_segments(disk->queue, 1);
1200 }
1201 #else
1202 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type)
1203 {
1204 }
1205 #endif /* CONFIG_BLK_DEV_INTEGRITY */
1206
1207 static void nvme_set_chunk_size(struct nvme_ns *ns)
1208 {
1209 u32 chunk_size = (((u32)ns->noiob) << (ns->lba_shift - 9));
1210 blk_queue_chunk_sectors(ns->queue, rounddown_pow_of_two(chunk_size));
1211 }
1212
1213 static void nvme_config_discard(struct nvme_ctrl *ctrl,
1214 unsigned stream_alignment, struct request_queue *queue)
1215 {
1216 u32 size = queue_logical_block_size(queue);
1217
1218 if (stream_alignment)
1219 size *= stream_alignment;
1220
1221 BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
1222 NVME_DSM_MAX_RANGES);
1223
1224 queue->limits.discard_alignment = size;
1225 queue->limits.discard_granularity = size;
1226
1227 blk_queue_max_discard_sectors(queue, UINT_MAX);
1228 blk_queue_max_discard_segments(queue, NVME_DSM_MAX_RANGES);
1229 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, queue);
1230
1231 if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
1232 blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1233 }
1234
1235 static void nvme_report_ns_ids(struct nvme_ctrl *ctrl, unsigned int nsid,
1236 struct nvme_id_ns *id, u8 *eui64, u8 *nguid, uuid_t *uuid)
1237 {
1238 if (ctrl->vs >= NVME_VS(1, 1, 0))
1239 memcpy(eui64, id->eui64, sizeof(id->eui64));
1240 if (ctrl->vs >= NVME_VS(1, 2, 0))
1241 memcpy(nguid, id->nguid, sizeof(id->nguid));
1242 if (ctrl->vs >= NVME_VS(1, 3, 0)) {
1243 /* Don't treat error as fatal we potentially
1244 * already have a NGUID or EUI-64
1245 */
1246 if (nvme_identify_ns_descs(ctrl, nsid, eui64, nguid, uuid))
1247 dev_warn(ctrl->device,
1248 "%s: Identify Descriptors failed\n", __func__);
1249 }
1250 }
1251
1252 static void nvme_update_disk_info(struct gendisk *disk,
1253 struct nvme_ns *ns, struct nvme_id_ns *id)
1254 {
1255 sector_t capacity = le64_to_cpup(&id->nsze) << (ns->lba_shift - 9);
1256 unsigned stream_alignment = 0;
1257
1258 if (ns->ctrl->nr_streams && ns->sws && ns->sgs)
1259 stream_alignment = ns->sws * ns->sgs;
1260
1261 blk_mq_freeze_queue(disk->queue);
1262 blk_integrity_unregister(disk);
1263
1264 blk_queue_logical_block_size(disk->queue, 1 << ns->lba_shift);
1265 if (ns->ms && !ns->ext &&
1266 (ns->ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
1267 nvme_init_integrity(disk, ns->ms, ns->pi_type);
1268 if (ns->ms && !nvme_ns_has_pi(ns) && !blk_get_integrity(disk))
1269 capacity = 0;
1270 set_capacity(disk, capacity);
1271
1272 if (ns->ctrl->oncs & NVME_CTRL_ONCS_DSM)
1273 nvme_config_discard(ns->ctrl, stream_alignment, disk->queue);
1274 blk_mq_unfreeze_queue(disk->queue);
1275 }
1276
1277 static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
1278 {
1279 struct nvme_ns *ns = disk->private_data;
1280
1281 /*
1282 * If identify namespace failed, use default 512 byte block size so
1283 * block layer can use before failing read/write for 0 capacity.
1284 */
1285 ns->lba_shift = id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ds;
1286 if (ns->lba_shift == 0)
1287 ns->lba_shift = 9;
1288 ns->noiob = le16_to_cpu(id->noiob);
1289 ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
1290 ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms);
1291 /* the PI implementation requires metadata equal t10 pi tuple size */
1292 if (ns->ms == sizeof(struct t10_pi_tuple))
1293 ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
1294 else
1295 ns->pi_type = 0;
1296
1297 if (ns->noiob)
1298 nvme_set_chunk_size(ns);
1299 nvme_update_disk_info(disk, ns, id);
1300 }
1301
1302 static int nvme_revalidate_disk(struct gendisk *disk)
1303 {
1304 struct nvme_ns *ns = disk->private_data;
1305 struct nvme_ctrl *ctrl = ns->ctrl;
1306 struct nvme_id_ns *id;
1307 u8 eui64[8] = { 0 }, nguid[16] = { 0 };
1308 uuid_t uuid = uuid_null;
1309 int ret = 0;
1310
1311 if (test_bit(NVME_NS_DEAD, &ns->flags)) {
1312 set_capacity(disk, 0);
1313 return -ENODEV;
1314 }
1315
1316 id = nvme_identify_ns(ctrl, ns->ns_id);
1317 if (!id)
1318 return -ENODEV;
1319
1320 if (id->ncap == 0) {
1321 ret = -ENODEV;
1322 goto out;
1323 }
1324
1325 nvme_report_ns_ids(ctrl, ns->ns_id, id, eui64, nguid, &uuid);
1326 if (!uuid_equal(&ns->uuid, &uuid) ||
1327 memcmp(&ns->nguid, &nguid, sizeof(ns->nguid)) ||
1328 memcmp(&ns->eui, &eui64, sizeof(ns->eui))) {
1329 dev_err(ctrl->device,
1330 "identifiers changed for nsid %d\n", ns->ns_id);
1331 ret = -ENODEV;
1332 }
1333
1334 out:
1335 kfree(id);
1336 return ret;
1337 }
1338
1339 static char nvme_pr_type(enum pr_type type)
1340 {
1341 switch (type) {
1342 case PR_WRITE_EXCLUSIVE:
1343 return 1;
1344 case PR_EXCLUSIVE_ACCESS:
1345 return 2;
1346 case PR_WRITE_EXCLUSIVE_REG_ONLY:
1347 return 3;
1348 case PR_EXCLUSIVE_ACCESS_REG_ONLY:
1349 return 4;
1350 case PR_WRITE_EXCLUSIVE_ALL_REGS:
1351 return 5;
1352 case PR_EXCLUSIVE_ACCESS_ALL_REGS:
1353 return 6;
1354 default:
1355 return 0;
1356 }
1357 };
1358
1359 static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
1360 u64 key, u64 sa_key, u8 op)
1361 {
1362 struct nvme_ns *ns = bdev->bd_disk->private_data;
1363 struct nvme_command c;
1364 u8 data[16] = { 0, };
1365
1366 put_unaligned_le64(key, &data[0]);
1367 put_unaligned_le64(sa_key, &data[8]);
1368
1369 memset(&c, 0, sizeof(c));
1370 c.common.opcode = op;
1371 c.common.nsid = cpu_to_le32(ns->ns_id);
1372 c.common.cdw10[0] = cpu_to_le32(cdw10);
1373
1374 return nvme_submit_sync_cmd(ns->queue, &c, data, 16);
1375 }
1376
1377 static int nvme_pr_register(struct block_device *bdev, u64 old,
1378 u64 new, unsigned flags)
1379 {
1380 u32 cdw10;
1381
1382 if (flags & ~PR_FL_IGNORE_KEY)
1383 return -EOPNOTSUPP;
1384
1385 cdw10 = old ? 2 : 0;
1386 cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
1387 cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
1388 return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
1389 }
1390
1391 static int nvme_pr_reserve(struct block_device *bdev, u64 key,
1392 enum pr_type type, unsigned flags)
1393 {
1394 u32 cdw10;
1395
1396 if (flags & ~PR_FL_IGNORE_KEY)
1397 return -EOPNOTSUPP;
1398
1399 cdw10 = nvme_pr_type(type) << 8;
1400 cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
1401 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
1402 }
1403
1404 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
1405 enum pr_type type, bool abort)
1406 {
1407 u32 cdw10 = nvme_pr_type(type) << 8 | abort ? 2 : 1;
1408 return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
1409 }
1410
1411 static int nvme_pr_clear(struct block_device *bdev, u64 key)
1412 {
1413 u32 cdw10 = 1 | (key ? 1 << 3 : 0);
1414 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
1415 }
1416
1417 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
1418 {
1419 u32 cdw10 = nvme_pr_type(type) << 8 | key ? 1 << 3 : 0;
1420 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
1421 }
1422
1423 static const struct pr_ops nvme_pr_ops = {
1424 .pr_register = nvme_pr_register,
1425 .pr_reserve = nvme_pr_reserve,
1426 .pr_release = nvme_pr_release,
1427 .pr_preempt = nvme_pr_preempt,
1428 .pr_clear = nvme_pr_clear,
1429 };
1430
1431 #ifdef CONFIG_BLK_SED_OPAL
1432 int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
1433 bool send)
1434 {
1435 struct nvme_ctrl *ctrl = data;
1436 struct nvme_command cmd;
1437
1438 memset(&cmd, 0, sizeof(cmd));
1439 if (send)
1440 cmd.common.opcode = nvme_admin_security_send;
1441 else
1442 cmd.common.opcode = nvme_admin_security_recv;
1443 cmd.common.nsid = 0;
1444 cmd.common.cdw10[0] = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
1445 cmd.common.cdw10[1] = cpu_to_le32(len);
1446
1447 return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
1448 ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0);
1449 }
1450 EXPORT_SYMBOL_GPL(nvme_sec_submit);
1451 #endif /* CONFIG_BLK_SED_OPAL */
1452
1453 static const struct block_device_operations nvme_fops = {
1454 .owner = THIS_MODULE,
1455 .ioctl = nvme_ioctl,
1456 .compat_ioctl = nvme_ioctl,
1457 .open = nvme_open,
1458 .release = nvme_release,
1459 .getgeo = nvme_getgeo,
1460 .revalidate_disk= nvme_revalidate_disk,
1461 .pr_ops = &nvme_pr_ops,
1462 };
1463
1464 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
1465 {
1466 unsigned long timeout =
1467 ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
1468 u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
1469 int ret;
1470
1471 while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
1472 if (csts == ~0)
1473 return -ENODEV;
1474 if ((csts & NVME_CSTS_RDY) == bit)
1475 break;
1476
1477 msleep(100);
1478 if (fatal_signal_pending(current))
1479 return -EINTR;
1480 if (time_after(jiffies, timeout)) {
1481 dev_err(ctrl->device,
1482 "Device not ready; aborting %s\n", enabled ?
1483 "initialisation" : "reset");
1484 return -ENODEV;
1485 }
1486 }
1487
1488 return ret;
1489 }
1490
1491 /*
1492 * If the device has been passed off to us in an enabled state, just clear
1493 * the enabled bit. The spec says we should set the 'shutdown notification
1494 * bits', but doing so may cause the device to complete commands to the
1495 * admin queue ... and we don't know what memory that might be pointing at!
1496 */
1497 int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
1498 {
1499 int ret;
1500
1501 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
1502 ctrl->ctrl_config &= ~NVME_CC_ENABLE;
1503
1504 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
1505 if (ret)
1506 return ret;
1507
1508 if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
1509 msleep(NVME_QUIRK_DELAY_AMOUNT);
1510
1511 return nvme_wait_ready(ctrl, cap, false);
1512 }
1513 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
1514
1515 int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
1516 {
1517 /*
1518 * Default to a 4K page size, with the intention to update this
1519 * path in the future to accomodate architectures with differing
1520 * kernel and IO page sizes.
1521 */
1522 unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12, page_shift = 12;
1523 int ret;
1524
1525 if (page_shift < dev_page_min) {
1526 dev_err(ctrl->device,
1527 "Minimum device page size %u too large for host (%u)\n",
1528 1 << dev_page_min, 1 << page_shift);
1529 return -ENODEV;
1530 }
1531
1532 ctrl->page_size = 1 << page_shift;
1533
1534 ctrl->ctrl_config = NVME_CC_CSS_NVM;
1535 ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
1536 ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
1537 ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
1538 ctrl->ctrl_config |= NVME_CC_ENABLE;
1539
1540 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
1541 if (ret)
1542 return ret;
1543 return nvme_wait_ready(ctrl, cap, true);
1544 }
1545 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
1546
1547 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
1548 {
1549 unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
1550 u32 csts;
1551 int ret;
1552
1553 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
1554 ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
1555
1556 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
1557 if (ret)
1558 return ret;
1559
1560 while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
1561 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
1562 break;
1563
1564 msleep(100);
1565 if (fatal_signal_pending(current))
1566 return -EINTR;
1567 if (time_after(jiffies, timeout)) {
1568 dev_err(ctrl->device,
1569 "Device shutdown incomplete; abort shutdown\n");
1570 return -ENODEV;
1571 }
1572 }
1573
1574 return ret;
1575 }
1576 EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
1577
1578 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
1579 struct request_queue *q)
1580 {
1581 bool vwc = false;
1582
1583 if (ctrl->max_hw_sectors) {
1584 u32 max_segments =
1585 (ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1;
1586
1587 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
1588 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
1589 }
1590 if (ctrl->quirks & NVME_QUIRK_STRIPE_SIZE)
1591 blk_queue_chunk_sectors(q, ctrl->max_hw_sectors);
1592 blk_queue_virt_boundary(q, ctrl->page_size - 1);
1593 if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
1594 vwc = true;
1595 blk_queue_write_cache(q, vwc, vwc);
1596 }
1597
1598 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
1599 {
1600 __le64 ts;
1601 int ret;
1602
1603 if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
1604 return 0;
1605
1606 ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
1607 ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
1608 NULL);
1609 if (ret)
1610 dev_warn_once(ctrl->device,
1611 "could not set timestamp (%d)\n", ret);
1612 return ret;
1613 }
1614
1615 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
1616 {
1617 /*
1618 * APST (Autonomous Power State Transition) lets us program a
1619 * table of power state transitions that the controller will
1620 * perform automatically. We configure it with a simple
1621 * heuristic: we are willing to spend at most 2% of the time
1622 * transitioning between power states. Therefore, when running
1623 * in any given state, we will enter the next lower-power
1624 * non-operational state after waiting 50 * (enlat + exlat)
1625 * microseconds, as long as that state's exit latency is under
1626 * the requested maximum latency.
1627 *
1628 * We will not autonomously enter any non-operational state for
1629 * which the total latency exceeds ps_max_latency_us. Users
1630 * can set ps_max_latency_us to zero to turn off APST.
1631 */
1632
1633 unsigned apste;
1634 struct nvme_feat_auto_pst *table;
1635 u64 max_lat_us = 0;
1636 int max_ps = -1;
1637 int ret;
1638
1639 /*
1640 * If APST isn't supported or if we haven't been initialized yet,
1641 * then don't do anything.
1642 */
1643 if (!ctrl->apsta)
1644 return 0;
1645
1646 if (ctrl->npss > 31) {
1647 dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
1648 return 0;
1649 }
1650
1651 table = kzalloc(sizeof(*table), GFP_KERNEL);
1652 if (!table)
1653 return 0;
1654
1655 if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
1656 /* Turn off APST. */
1657 apste = 0;
1658 dev_dbg(ctrl->device, "APST disabled\n");
1659 } else {
1660 __le64 target = cpu_to_le64(0);
1661 int state;
1662
1663 /*
1664 * Walk through all states from lowest- to highest-power.
1665 * According to the spec, lower-numbered states use more
1666 * power. NPSS, despite the name, is the index of the
1667 * lowest-power state, not the number of states.
1668 */
1669 for (state = (int)ctrl->npss; state >= 0; state--) {
1670 u64 total_latency_us, exit_latency_us, transition_ms;
1671
1672 if (target)
1673 table->entries[state] = target;
1674
1675 /*
1676 * Don't allow transitions to the deepest state
1677 * if it's quirked off.
1678 */
1679 if (state == ctrl->npss &&
1680 (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
1681 continue;
1682
1683 /*
1684 * Is this state a useful non-operational state for
1685 * higher-power states to autonomously transition to?
1686 */
1687 if (!(ctrl->psd[state].flags &
1688 NVME_PS_FLAGS_NON_OP_STATE))
1689 continue;
1690
1691 exit_latency_us =
1692 (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
1693 if (exit_latency_us > ctrl->ps_max_latency_us)
1694 continue;
1695
1696 total_latency_us =
1697 exit_latency_us +
1698 le32_to_cpu(ctrl->psd[state].entry_lat);
1699
1700 /*
1701 * This state is good. Use it as the APST idle
1702 * target for higher power states.
1703 */
1704 transition_ms = total_latency_us + 19;
1705 do_div(transition_ms, 20);
1706 if (transition_ms > (1 << 24) - 1)
1707 transition_ms = (1 << 24) - 1;
1708
1709 target = cpu_to_le64((state << 3) |
1710 (transition_ms << 8));
1711
1712 if (max_ps == -1)
1713 max_ps = state;
1714
1715 if (total_latency_us > max_lat_us)
1716 max_lat_us = total_latency_us;
1717 }
1718
1719 apste = 1;
1720
1721 if (max_ps == -1) {
1722 dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
1723 } else {
1724 dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
1725 max_ps, max_lat_us, (int)sizeof(*table), table);
1726 }
1727 }
1728
1729 ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
1730 table, sizeof(*table), NULL);
1731 if (ret)
1732 dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
1733
1734 kfree(table);
1735 return ret;
1736 }
1737
1738 static void nvme_set_latency_tolerance(struct device *dev, s32 val)
1739 {
1740 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
1741 u64 latency;
1742
1743 switch (val) {
1744 case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
1745 case PM_QOS_LATENCY_ANY:
1746 latency = U64_MAX;
1747 break;
1748
1749 default:
1750 latency = val;
1751 }
1752
1753 if (ctrl->ps_max_latency_us != latency) {
1754 ctrl->ps_max_latency_us = latency;
1755 nvme_configure_apst(ctrl);
1756 }
1757 }
1758
1759 struct nvme_core_quirk_entry {
1760 /*
1761 * NVMe model and firmware strings are padded with spaces. For
1762 * simplicity, strings in the quirk table are padded with NULLs
1763 * instead.
1764 */
1765 u16 vid;
1766 const char *mn;
1767 const char *fr;
1768 unsigned long quirks;
1769 };
1770
1771 static const struct nvme_core_quirk_entry core_quirks[] = {
1772 {
1773 /*
1774 * This Toshiba device seems to die using any APST states. See:
1775 * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
1776 */
1777 .vid = 0x1179,
1778 .mn = "THNSF5256GPUK TOSHIBA",
1779 .quirks = NVME_QUIRK_NO_APST,
1780 }
1781 };
1782
1783 /* match is null-terminated but idstr is space-padded. */
1784 static bool string_matches(const char *idstr, const char *match, size_t len)
1785 {
1786 size_t matchlen;
1787
1788 if (!match)
1789 return true;
1790
1791 matchlen = strlen(match);
1792 WARN_ON_ONCE(matchlen > len);
1793
1794 if (memcmp(idstr, match, matchlen))
1795 return false;
1796
1797 for (; matchlen < len; matchlen++)
1798 if (idstr[matchlen] != ' ')
1799 return false;
1800
1801 return true;
1802 }
1803
1804 static bool quirk_matches(const struct nvme_id_ctrl *id,
1805 const struct nvme_core_quirk_entry *q)
1806 {
1807 return q->vid == le16_to_cpu(id->vid) &&
1808 string_matches(id->mn, q->mn, sizeof(id->mn)) &&
1809 string_matches(id->fr, q->fr, sizeof(id->fr));
1810 }
1811
1812 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
1813 struct nvme_id_ctrl *id)
1814 {
1815 size_t nqnlen;
1816 int off;
1817
1818 nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
1819 if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
1820 strncpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
1821 return;
1822 }
1823
1824 if (ctrl->vs >= NVME_VS(1, 2, 1))
1825 dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
1826
1827 /* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */
1828 off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
1829 "nqn.2014.08.org.nvmexpress:%4x%4x",
1830 le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
1831 memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
1832 off += sizeof(id->sn);
1833 memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
1834 off += sizeof(id->mn);
1835 memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
1836 }
1837
1838 static void __nvme_release_subsystem(struct nvme_subsystem *subsys)
1839 {
1840 ida_simple_remove(&nvme_subsystems_ida, subsys->instance);
1841 kfree(subsys);
1842 }
1843
1844 static void nvme_release_subsystem(struct device *dev)
1845 {
1846 __nvme_release_subsystem(container_of(dev, struct nvme_subsystem, dev));
1847 }
1848
1849 static void nvme_destroy_subsystem(struct kref *ref)
1850 {
1851 struct nvme_subsystem *subsys =
1852 container_of(ref, struct nvme_subsystem, ref);
1853
1854 mutex_lock(&nvme_subsystems_lock);
1855 list_del(&subsys->entry);
1856 mutex_unlock(&nvme_subsystems_lock);
1857
1858 device_del(&subsys->dev);
1859 put_device(&subsys->dev);
1860 }
1861
1862 static void nvme_put_subsystem(struct nvme_subsystem *subsys)
1863 {
1864 kref_put(&subsys->ref, nvme_destroy_subsystem);
1865 }
1866
1867 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
1868 {
1869 struct nvme_subsystem *subsys;
1870
1871 lockdep_assert_held(&nvme_subsystems_lock);
1872
1873 list_for_each_entry(subsys, &nvme_subsystems, entry) {
1874 if (strcmp(subsys->subnqn, subsysnqn))
1875 continue;
1876 if (!kref_get_unless_zero(&subsys->ref))
1877 continue;
1878 return subsys;
1879 }
1880
1881 return NULL;
1882 }
1883
1884 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
1885 {
1886 struct nvme_subsystem *subsys, *found;
1887 int ret;
1888
1889 subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
1890 if (!subsys)
1891 return -ENOMEM;
1892 ret = ida_simple_get(&nvme_subsystems_ida, 0, 0, GFP_KERNEL);
1893 if (ret < 0) {
1894 kfree(subsys);
1895 return ret;
1896 }
1897 subsys->instance = ret;
1898 mutex_init(&subsys->lock);
1899 kref_init(&subsys->ref);
1900 INIT_LIST_HEAD(&subsys->ctrls);
1901 nvme_init_subnqn(subsys, ctrl, id);
1902 memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
1903 memcpy(subsys->model, id->mn, sizeof(subsys->model));
1904 memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev));
1905 subsys->vendor_id = le16_to_cpu(id->vid);
1906 subsys->cmic = id->cmic;
1907
1908 subsys->dev.class = nvme_subsys_class;
1909 subsys->dev.release = nvme_release_subsystem;
1910 dev_set_name(&subsys->dev, "nvme-subsys%d", subsys->instance);
1911 device_initialize(&subsys->dev);
1912
1913 mutex_lock(&nvme_subsystems_lock);
1914 found = __nvme_find_get_subsystem(subsys->subnqn);
1915 if (found) {
1916 /*
1917 * Verify that the subsystem actually supports multiple
1918 * controllers, else bail out.
1919 */
1920 if (!(id->cmic & (1 << 1))) {
1921 dev_err(ctrl->device,
1922 "ignoring ctrl due to duplicate subnqn (%s).\n",
1923 found->subnqn);
1924 nvme_put_subsystem(found);
1925 ret = -EINVAL;
1926 goto out_unlock;
1927 }
1928
1929 __nvme_release_subsystem(subsys);
1930 subsys = found;
1931 } else {
1932 ret = device_add(&subsys->dev);
1933 if (ret) {
1934 dev_err(ctrl->device,
1935 "failed to register subsystem device.\n");
1936 goto out_unlock;
1937 }
1938 list_add_tail(&subsys->entry, &nvme_subsystems);
1939 }
1940
1941 ctrl->subsys = subsys;
1942 mutex_unlock(&nvme_subsystems_lock);
1943
1944 if (sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
1945 dev_name(ctrl->device))) {
1946 dev_err(ctrl->device,
1947 "failed to create sysfs link from subsystem.\n");
1948 /* the transport driver will eventually put the subsystem */
1949 return -EINVAL;
1950 }
1951
1952 mutex_lock(&subsys->lock);
1953 list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
1954 mutex_unlock(&subsys->lock);
1955
1956 return 0;
1957
1958 out_unlock:
1959 mutex_unlock(&nvme_subsystems_lock);
1960 put_device(&subsys->dev);
1961 return ret;
1962 }
1963
1964 static int nvme_get_log(struct nvme_ctrl *ctrl, u8 log_page, void *log,
1965 size_t size)
1966 {
1967 struct nvme_command c = { };
1968
1969 c.common.opcode = nvme_admin_get_log_page;
1970 c.common.nsid = cpu_to_le32(NVME_NSID_ALL);
1971 c.common.cdw10[0] = nvme_get_log_dw10(log_page, size);
1972
1973 return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
1974 }
1975
1976 static int nvme_get_effects_log(struct nvme_ctrl *ctrl)
1977 {
1978 int ret;
1979
1980 if (!ctrl->effects)
1981 ctrl->effects = kzalloc(sizeof(*ctrl->effects), GFP_KERNEL);
1982
1983 if (!ctrl->effects)
1984 return 0;
1985
1986 ret = nvme_get_log(ctrl, NVME_LOG_CMD_EFFECTS, ctrl->effects,
1987 sizeof(*ctrl->effects));
1988 if (ret) {
1989 kfree(ctrl->effects);
1990 ctrl->effects = NULL;
1991 }
1992 return ret;
1993 }
1994
1995 /*
1996 * Initialize the cached copies of the Identify data and various controller
1997 * register in our nvme_ctrl structure. This should be called as soon as
1998 * the admin queue is fully up and running.
1999 */
2000 int nvme_init_identify(struct nvme_ctrl *ctrl)
2001 {
2002 struct nvme_id_ctrl *id;
2003 u64 cap;
2004 int ret, page_shift;
2005 u32 max_hw_sectors;
2006 bool prev_apst_enabled;
2007
2008 ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
2009 if (ret) {
2010 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
2011 return ret;
2012 }
2013
2014 ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &cap);
2015 if (ret) {
2016 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2017 return ret;
2018 }
2019 page_shift = NVME_CAP_MPSMIN(cap) + 12;
2020
2021 if (ctrl->vs >= NVME_VS(1, 1, 0))
2022 ctrl->subsystem = NVME_CAP_NSSRC(cap);
2023
2024 ret = nvme_identify_ctrl(ctrl, &id);
2025 if (ret) {
2026 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
2027 return -EIO;
2028 }
2029
2030 if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
2031 ret = nvme_get_effects_log(ctrl);
2032 if (ret < 0)
2033 return ret;
2034 }
2035
2036 if (!ctrl->identified) {
2037 int i;
2038
2039 ret = nvme_init_subsystem(ctrl, id);
2040 if (ret)
2041 goto out_free;
2042
2043 /*
2044 * Check for quirks. Quirk can depend on firmware version,
2045 * so, in principle, the set of quirks present can change
2046 * across a reset. As a possible future enhancement, we
2047 * could re-scan for quirks every time we reinitialize
2048 * the device, but we'd have to make sure that the driver
2049 * behaves intelligently if the quirks change.
2050 */
2051 for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
2052 if (quirk_matches(id, &core_quirks[i]))
2053 ctrl->quirks |= core_quirks[i].quirks;
2054 }
2055 }
2056
2057 if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
2058 dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
2059 ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
2060 }
2061
2062 ctrl->oacs = le16_to_cpu(id->oacs);
2063 ctrl->oncs = le16_to_cpup(&id->oncs);
2064 atomic_set(&ctrl->abort_limit, id->acl + 1);
2065 ctrl->vwc = id->vwc;
2066 ctrl->cntlid = le16_to_cpup(&id->cntlid);
2067 if (id->mdts)
2068 max_hw_sectors = 1 << (id->mdts + page_shift - 9);
2069 else
2070 max_hw_sectors = UINT_MAX;
2071 ctrl->max_hw_sectors =
2072 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
2073
2074 nvme_set_queue_limits(ctrl, ctrl->admin_q);
2075 ctrl->sgls = le32_to_cpu(id->sgls);
2076 ctrl->kas = le16_to_cpu(id->kas);
2077
2078 if (id->rtd3e) {
2079 /* us -> s */
2080 u32 transition_time = le32_to_cpu(id->rtd3e) / 1000000;
2081
2082 ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
2083 shutdown_timeout, 60);
2084
2085 if (ctrl->shutdown_timeout != shutdown_timeout)
2086 dev_warn(ctrl->device,
2087 "Shutdown timeout set to %u seconds\n",
2088 ctrl->shutdown_timeout);
2089 } else
2090 ctrl->shutdown_timeout = shutdown_timeout;
2091
2092 ctrl->npss = id->npss;
2093 ctrl->apsta = id->apsta;
2094 prev_apst_enabled = ctrl->apst_enabled;
2095 if (ctrl->quirks & NVME_QUIRK_NO_APST) {
2096 if (force_apst && id->apsta) {
2097 dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
2098 ctrl->apst_enabled = true;
2099 } else {
2100 ctrl->apst_enabled = false;
2101 }
2102 } else {
2103 ctrl->apst_enabled = id->apsta;
2104 }
2105 memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
2106
2107 if (ctrl->ops->flags & NVME_F_FABRICS) {
2108 ctrl->icdoff = le16_to_cpu(id->icdoff);
2109 ctrl->ioccsz = le32_to_cpu(id->ioccsz);
2110 ctrl->iorcsz = le32_to_cpu(id->iorcsz);
2111 ctrl->maxcmd = le16_to_cpu(id->maxcmd);
2112
2113 /*
2114 * In fabrics we need to verify the cntlid matches the
2115 * admin connect
2116 */
2117 if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
2118 ret = -EINVAL;
2119 goto out_free;
2120 }
2121
2122 if (!ctrl->opts->discovery_nqn && !ctrl->kas) {
2123 dev_err(ctrl->device,
2124 "keep-alive support is mandatory for fabrics\n");
2125 ret = -EINVAL;
2126 goto out_free;
2127 }
2128 } else {
2129 ctrl->cntlid = le16_to_cpu(id->cntlid);
2130 ctrl->hmpre = le32_to_cpu(id->hmpre);
2131 ctrl->hmmin = le32_to_cpu(id->hmmin);
2132 ctrl->hmminds = le32_to_cpu(id->hmminds);
2133 ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
2134 }
2135
2136 kfree(id);
2137
2138 if (ctrl->apst_enabled && !prev_apst_enabled)
2139 dev_pm_qos_expose_latency_tolerance(ctrl->device);
2140 else if (!ctrl->apst_enabled && prev_apst_enabled)
2141 dev_pm_qos_hide_latency_tolerance(ctrl->device);
2142
2143 ret = nvme_configure_apst(ctrl);
2144 if (ret < 0)
2145 return ret;
2146
2147 ret = nvme_configure_timestamp(ctrl);
2148 if (ret < 0)
2149 return ret;
2150
2151 ret = nvme_configure_directives(ctrl);
2152 if (ret < 0)
2153 return ret;
2154
2155 ctrl->identified = true;
2156
2157 return 0;
2158
2159 out_free:
2160 kfree(id);
2161 return ret;
2162 }
2163 EXPORT_SYMBOL_GPL(nvme_init_identify);
2164
2165 static int nvme_dev_open(struct inode *inode, struct file *file)
2166 {
2167 struct nvme_ctrl *ctrl =
2168 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
2169
2170 if (ctrl->state != NVME_CTRL_LIVE)
2171 return -EWOULDBLOCK;
2172 file->private_data = ctrl;
2173 return 0;
2174 }
2175
2176 static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
2177 {
2178 struct nvme_ns *ns;
2179 int ret;
2180
2181 mutex_lock(&ctrl->namespaces_mutex);
2182 if (list_empty(&ctrl->namespaces)) {
2183 ret = -ENOTTY;
2184 goto out_unlock;
2185 }
2186
2187 ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
2188 if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
2189 dev_warn(ctrl->device,
2190 "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
2191 ret = -EINVAL;
2192 goto out_unlock;
2193 }
2194
2195 dev_warn(ctrl->device,
2196 "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
2197 kref_get(&ns->kref);
2198 mutex_unlock(&ctrl->namespaces_mutex);
2199
2200 ret = nvme_user_cmd(ctrl, ns, argp);
2201 nvme_put_ns(ns);
2202 return ret;
2203
2204 out_unlock:
2205 mutex_unlock(&ctrl->namespaces_mutex);
2206 return ret;
2207 }
2208
2209 static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
2210 unsigned long arg)
2211 {
2212 struct nvme_ctrl *ctrl = file->private_data;
2213 void __user *argp = (void __user *)arg;
2214
2215 switch (cmd) {
2216 case NVME_IOCTL_ADMIN_CMD:
2217 return nvme_user_cmd(ctrl, NULL, argp);
2218 case NVME_IOCTL_IO_CMD:
2219 return nvme_dev_user_cmd(ctrl, argp);
2220 case NVME_IOCTL_RESET:
2221 dev_warn(ctrl->device, "resetting controller\n");
2222 return nvme_reset_ctrl_sync(ctrl);
2223 case NVME_IOCTL_SUBSYS_RESET:
2224 return nvme_reset_subsystem(ctrl);
2225 case NVME_IOCTL_RESCAN:
2226 nvme_queue_scan(ctrl);
2227 return 0;
2228 default:
2229 return -ENOTTY;
2230 }
2231 }
2232
2233 static const struct file_operations nvme_dev_fops = {
2234 .owner = THIS_MODULE,
2235 .open = nvme_dev_open,
2236 .unlocked_ioctl = nvme_dev_ioctl,
2237 .compat_ioctl = nvme_dev_ioctl,
2238 };
2239
2240 static ssize_t nvme_sysfs_reset(struct device *dev,
2241 struct device_attribute *attr, const char *buf,
2242 size_t count)
2243 {
2244 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2245 int ret;
2246
2247 ret = nvme_reset_ctrl_sync(ctrl);
2248 if (ret < 0)
2249 return ret;
2250 return count;
2251 }
2252 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
2253
2254 static ssize_t nvme_sysfs_rescan(struct device *dev,
2255 struct device_attribute *attr, const char *buf,
2256 size_t count)
2257 {
2258 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2259
2260 nvme_queue_scan(ctrl);
2261 return count;
2262 }
2263 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
2264
2265 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
2266 char *buf)
2267 {
2268 struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
2269 struct nvme_subsystem *subsys = ns->ctrl->subsys;
2270 int serial_len = sizeof(subsys->serial);
2271 int model_len = sizeof(subsys->model);
2272
2273 if (!uuid_is_null(&ns->uuid))
2274 return sprintf(buf, "uuid.%pU\n", &ns->uuid);
2275
2276 if (memchr_inv(ns->nguid, 0, sizeof(ns->nguid)))
2277 return sprintf(buf, "eui.%16phN\n", ns->nguid);
2278
2279 if (memchr_inv(ns->eui, 0, sizeof(ns->eui)))
2280 return sprintf(buf, "eui.%8phN\n", ns->eui);
2281
2282 while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
2283 subsys->serial[serial_len - 1] == '\0'))
2284 serial_len--;
2285 while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
2286 subsys->model[model_len - 1] == '\0'))
2287 model_len--;
2288
2289 return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
2290 serial_len, subsys->serial, model_len, subsys->model,
2291 ns->ns_id);
2292 }
2293 static DEVICE_ATTR(wwid, S_IRUGO, wwid_show, NULL);
2294
2295 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
2296 char *buf)
2297 {
2298 struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
2299 return sprintf(buf, "%pU\n", ns->nguid);
2300 }
2301 static DEVICE_ATTR(nguid, S_IRUGO, nguid_show, NULL);
2302
2303 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
2304 char *buf)
2305 {
2306 struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
2307
2308 /* For backward compatibility expose the NGUID to userspace if
2309 * we have no UUID set
2310 */
2311 if (uuid_is_null(&ns->uuid)) {
2312 printk_ratelimited(KERN_WARNING
2313 "No UUID available providing old NGUID\n");
2314 return sprintf(buf, "%pU\n", ns->nguid);
2315 }
2316 return sprintf(buf, "%pU\n", &ns->uuid);
2317 }
2318 static DEVICE_ATTR(uuid, S_IRUGO, uuid_show, NULL);
2319
2320 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
2321 char *buf)
2322 {
2323 struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
2324 return sprintf(buf, "%8ph\n", ns->eui);
2325 }
2326 static DEVICE_ATTR(eui, S_IRUGO, eui_show, NULL);
2327
2328 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
2329 char *buf)
2330 {
2331 struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
2332 return sprintf(buf, "%d\n", ns->ns_id);
2333 }
2334 static DEVICE_ATTR(nsid, S_IRUGO, nsid_show, NULL);
2335
2336 static struct attribute *nvme_ns_attrs[] = {
2337 &dev_attr_wwid.attr,
2338 &dev_attr_uuid.attr,
2339 &dev_attr_nguid.attr,
2340 &dev_attr_eui.attr,
2341 &dev_attr_nsid.attr,
2342 NULL,
2343 };
2344
2345 static umode_t nvme_ns_attrs_are_visible(struct kobject *kobj,
2346 struct attribute *a, int n)
2347 {
2348 struct device *dev = container_of(kobj, struct device, kobj);
2349 struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
2350
2351 if (a == &dev_attr_uuid.attr) {
2352 if (uuid_is_null(&ns->uuid) ||
2353 !memchr_inv(ns->nguid, 0, sizeof(ns->nguid)))
2354 return 0;
2355 }
2356 if (a == &dev_attr_nguid.attr) {
2357 if (!memchr_inv(ns->nguid, 0, sizeof(ns->nguid)))
2358 return 0;
2359 }
2360 if (a == &dev_attr_eui.attr) {
2361 if (!memchr_inv(ns->eui, 0, sizeof(ns->eui)))
2362 return 0;
2363 }
2364 return a->mode;
2365 }
2366
2367 static const struct attribute_group nvme_ns_attr_group = {
2368 .attrs = nvme_ns_attrs,
2369 .is_visible = nvme_ns_attrs_are_visible,
2370 };
2371
2372 #define nvme_show_str_function(field) \
2373 static ssize_t field##_show(struct device *dev, \
2374 struct device_attribute *attr, char *buf) \
2375 { \
2376 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \
2377 return sprintf(buf, "%.*s\n", \
2378 (int)sizeof(ctrl->subsys->field), ctrl->subsys->field); \
2379 } \
2380 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
2381
2382 nvme_show_str_function(model);
2383 nvme_show_str_function(serial);
2384 nvme_show_str_function(firmware_rev);
2385
2386 #define nvme_show_int_function(field) \
2387 static ssize_t field##_show(struct device *dev, \
2388 struct device_attribute *attr, char *buf) \
2389 { \
2390 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \
2391 return sprintf(buf, "%d\n", ctrl->field); \
2392 } \
2393 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
2394
2395 nvme_show_int_function(cntlid);
2396
2397 static ssize_t nvme_sysfs_delete(struct device *dev,
2398 struct device_attribute *attr, const char *buf,
2399 size_t count)
2400 {
2401 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2402
2403 if (device_remove_file_self(dev, attr))
2404 nvme_delete_ctrl_sync(ctrl);
2405 return count;
2406 }
2407 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
2408
2409 static ssize_t nvme_sysfs_show_transport(struct device *dev,
2410 struct device_attribute *attr,
2411 char *buf)
2412 {
2413 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2414
2415 return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name);
2416 }
2417 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
2418
2419 static ssize_t nvme_sysfs_show_state(struct device *dev,
2420 struct device_attribute *attr,
2421 char *buf)
2422 {
2423 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2424 static const char *const state_name[] = {
2425 [NVME_CTRL_NEW] = "new",
2426 [NVME_CTRL_LIVE] = "live",
2427 [NVME_CTRL_RESETTING] = "resetting",
2428 [NVME_CTRL_RECONNECTING]= "reconnecting",
2429 [NVME_CTRL_DELETING] = "deleting",
2430 [NVME_CTRL_DEAD] = "dead",
2431 };
2432
2433 if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
2434 state_name[ctrl->state])
2435 return sprintf(buf, "%s\n", state_name[ctrl->state]);
2436
2437 return sprintf(buf, "unknown state\n");
2438 }
2439
2440 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
2441
2442 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
2443 struct device_attribute *attr,
2444 char *buf)
2445 {
2446 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2447
2448 return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->subsys->subnqn);
2449 }
2450 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
2451
2452 static ssize_t nvme_sysfs_show_address(struct device *dev,
2453 struct device_attribute *attr,
2454 char *buf)
2455 {
2456 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2457
2458 return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
2459 }
2460 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
2461
2462 static struct attribute *nvme_dev_attrs[] = {
2463 &dev_attr_reset_controller.attr,
2464 &dev_attr_rescan_controller.attr,
2465 &dev_attr_model.attr,
2466 &dev_attr_serial.attr,
2467 &dev_attr_firmware_rev.attr,
2468 &dev_attr_cntlid.attr,
2469 &dev_attr_delete_controller.attr,
2470 &dev_attr_transport.attr,
2471 &dev_attr_subsysnqn.attr,
2472 &dev_attr_address.attr,
2473 &dev_attr_state.attr,
2474 NULL
2475 };
2476
2477 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
2478 struct attribute *a, int n)
2479 {
2480 struct device *dev = container_of(kobj, struct device, kobj);
2481 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2482
2483 if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
2484 return 0;
2485 if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
2486 return 0;
2487
2488 return a->mode;
2489 }
2490
2491 static struct attribute_group nvme_dev_attrs_group = {
2492 .attrs = nvme_dev_attrs,
2493 .is_visible = nvme_dev_attrs_are_visible,
2494 };
2495
2496 static const struct attribute_group *nvme_dev_attr_groups[] = {
2497 &nvme_dev_attrs_group,
2498 NULL,
2499 };
2500
2501 static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
2502 {
2503 struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
2504 struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
2505
2506 return nsa->ns_id - nsb->ns_id;
2507 }
2508
2509 static struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
2510 {
2511 struct nvme_ns *ns, *ret = NULL;
2512
2513 mutex_lock(&ctrl->namespaces_mutex);
2514 list_for_each_entry(ns, &ctrl->namespaces, list) {
2515 if (ns->ns_id == nsid) {
2516 if (!kref_get_unless_zero(&ns->kref))
2517 continue;
2518 ret = ns;
2519 break;
2520 }
2521 if (ns->ns_id > nsid)
2522 break;
2523 }
2524 mutex_unlock(&ctrl->namespaces_mutex);
2525 return ret;
2526 }
2527
2528 static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
2529 {
2530 struct streams_directive_params s;
2531 int ret;
2532
2533 if (!ctrl->nr_streams)
2534 return 0;
2535
2536 ret = nvme_get_stream_params(ctrl, &s, ns->ns_id);
2537 if (ret)
2538 return ret;
2539
2540 ns->sws = le32_to_cpu(s.sws);
2541 ns->sgs = le16_to_cpu(s.sgs);
2542
2543 if (ns->sws) {
2544 unsigned int bs = 1 << ns->lba_shift;
2545
2546 blk_queue_io_min(ns->queue, bs * ns->sws);
2547 if (ns->sgs)
2548 blk_queue_io_opt(ns->queue, bs * ns->sws * ns->sgs);
2549 }
2550
2551 return 0;
2552 }
2553
2554 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
2555 {
2556 struct nvme_ns *ns;
2557 struct gendisk *disk;
2558 struct nvme_id_ns *id;
2559 char disk_name[DISK_NAME_LEN];
2560 int node = dev_to_node(ctrl->dev);
2561
2562 ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
2563 if (!ns)
2564 return;
2565
2566 ns->instance = ida_simple_get(&ctrl->ns_ida, 1, 0, GFP_KERNEL);
2567 if (ns->instance < 0)
2568 goto out_free_ns;
2569
2570 ns->queue = blk_mq_init_queue(ctrl->tagset);
2571 if (IS_ERR(ns->queue))
2572 goto out_release_instance;
2573 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
2574 ns->queue->queuedata = ns;
2575 ns->ctrl = ctrl;
2576
2577 kref_init(&ns->kref);
2578 ns->ns_id = nsid;
2579 ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
2580
2581 blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
2582 nvme_set_queue_limits(ctrl, ns->queue);
2583 nvme_setup_streams_ns(ctrl, ns);
2584
2585 sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->instance);
2586
2587 id = nvme_identify_ns(ctrl, nsid);
2588 if (!id)
2589 goto out_free_queue;
2590
2591 if (id->ncap == 0)
2592 goto out_free_id;
2593
2594 nvme_report_ns_ids(ctrl, ns->ns_id, id, ns->eui, ns->nguid, &ns->uuid);
2595
2596 if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
2597 if (nvme_nvm_register(ns, disk_name, node)) {
2598 dev_warn(ctrl->device, "LightNVM init failure\n");
2599 goto out_free_id;
2600 }
2601 }
2602
2603 disk = alloc_disk_node(0, node);
2604 if (!disk)
2605 goto out_free_id;
2606
2607 disk->fops = &nvme_fops;
2608 disk->private_data = ns;
2609 disk->queue = ns->queue;
2610 disk->flags = GENHD_FL_EXT_DEVT;
2611 memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
2612 ns->disk = disk;
2613
2614 __nvme_revalidate_disk(disk, id);
2615
2616 mutex_lock(&ctrl->namespaces_mutex);
2617 list_add_tail(&ns->list, &ctrl->namespaces);
2618 mutex_unlock(&ctrl->namespaces_mutex);
2619
2620 nvme_get_ctrl(ctrl);
2621
2622 kfree(id);
2623
2624 device_add_disk(ctrl->device, ns->disk);
2625 if (sysfs_create_group(&disk_to_dev(ns->disk)->kobj,
2626 &nvme_ns_attr_group))
2627 pr_warn("%s: failed to create sysfs group for identification\n",
2628 ns->disk->disk_name);
2629 if (ns->ndev && nvme_nvm_register_sysfs(ns))
2630 pr_warn("%s: failed to register lightnvm sysfs group for identification\n",
2631 ns->disk->disk_name);
2632 return;
2633 out_free_id:
2634 kfree(id);
2635 out_free_queue:
2636 blk_cleanup_queue(ns->queue);
2637 out_release_instance:
2638 ida_simple_remove(&ctrl->ns_ida, ns->instance);
2639 out_free_ns:
2640 kfree(ns);
2641 }
2642
2643 static void nvme_ns_remove(struct nvme_ns *ns)
2644 {
2645 if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
2646 return;
2647
2648 if (ns->disk && ns->disk->flags & GENHD_FL_UP) {
2649 if (blk_get_integrity(ns->disk))
2650 blk_integrity_unregister(ns->disk);
2651 sysfs_remove_group(&disk_to_dev(ns->disk)->kobj,
2652 &nvme_ns_attr_group);
2653 if (ns->ndev)
2654 nvme_nvm_unregister_sysfs(ns);
2655 del_gendisk(ns->disk);
2656 blk_cleanup_queue(ns->queue);
2657 }
2658
2659 mutex_lock(&ns->ctrl->namespaces_mutex);
2660 list_del_init(&ns->list);
2661 mutex_unlock(&ns->ctrl->namespaces_mutex);
2662
2663 nvme_put_ns(ns);
2664 }
2665
2666 static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
2667 {
2668 struct nvme_ns *ns;
2669
2670 ns = nvme_find_get_ns(ctrl, nsid);
2671 if (ns) {
2672 if (ns->disk && revalidate_disk(ns->disk))
2673 nvme_ns_remove(ns);
2674 nvme_put_ns(ns);
2675 } else
2676 nvme_alloc_ns(ctrl, nsid);
2677 }
2678
2679 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
2680 unsigned nsid)
2681 {
2682 struct nvme_ns *ns, *next;
2683
2684 list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
2685 if (ns->ns_id > nsid)
2686 nvme_ns_remove(ns);
2687 }
2688 }
2689
2690 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn)
2691 {
2692 struct nvme_ns *ns;
2693 __le32 *ns_list;
2694 unsigned i, j, nsid, prev = 0, num_lists = DIV_ROUND_UP(nn, 1024);
2695 int ret = 0;
2696
2697 ns_list = kzalloc(0x1000, GFP_KERNEL);
2698 if (!ns_list)
2699 return -ENOMEM;
2700
2701 for (i = 0; i < num_lists; i++) {
2702 ret = nvme_identify_ns_list(ctrl, prev, ns_list);
2703 if (ret)
2704 goto free;
2705
2706 for (j = 0; j < min(nn, 1024U); j++) {
2707 nsid = le32_to_cpu(ns_list[j]);
2708 if (!nsid)
2709 goto out;
2710
2711 nvme_validate_ns(ctrl, nsid);
2712
2713 while (++prev < nsid) {
2714 ns = nvme_find_get_ns(ctrl, prev);
2715 if (ns) {
2716 nvme_ns_remove(ns);
2717 nvme_put_ns(ns);
2718 }
2719 }
2720 }
2721 nn -= j;
2722 }
2723 out:
2724 nvme_remove_invalid_namespaces(ctrl, prev);
2725 free:
2726 kfree(ns_list);
2727 return ret;
2728 }
2729
2730 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl, unsigned nn)
2731 {
2732 unsigned i;
2733
2734 for (i = 1; i <= nn; i++)
2735 nvme_validate_ns(ctrl, i);
2736
2737 nvme_remove_invalid_namespaces(ctrl, nn);
2738 }
2739
2740 static void nvme_scan_work(struct work_struct *work)
2741 {
2742 struct nvme_ctrl *ctrl =
2743 container_of(work, struct nvme_ctrl, scan_work);
2744 struct nvme_id_ctrl *id;
2745 unsigned nn;
2746
2747 if (ctrl->state != NVME_CTRL_LIVE)
2748 return;
2749
2750 if (nvme_identify_ctrl(ctrl, &id))
2751 return;
2752
2753 nn = le32_to_cpu(id->nn);
2754 if (ctrl->vs >= NVME_VS(1, 1, 0) &&
2755 !(ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)) {
2756 if (!nvme_scan_ns_list(ctrl, nn))
2757 goto done;
2758 }
2759 nvme_scan_ns_sequential(ctrl, nn);
2760 done:
2761 mutex_lock(&ctrl->namespaces_mutex);
2762 list_sort(NULL, &ctrl->namespaces, ns_cmp);
2763 mutex_unlock(&ctrl->namespaces_mutex);
2764 kfree(id);
2765 }
2766
2767 void nvme_queue_scan(struct nvme_ctrl *ctrl)
2768 {
2769 /*
2770 * Do not queue new scan work when a controller is reset during
2771 * removal.
2772 */
2773 if (ctrl->state == NVME_CTRL_LIVE)
2774 queue_work(nvme_wq, &ctrl->scan_work);
2775 }
2776 EXPORT_SYMBOL_GPL(nvme_queue_scan);
2777
2778 /*
2779 * This function iterates the namespace list unlocked to allow recovery from
2780 * controller failure. It is up to the caller to ensure the namespace list is
2781 * not modified by scan work while this function is executing.
2782 */
2783 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
2784 {
2785 struct nvme_ns *ns, *next;
2786
2787 /*
2788 * The dead states indicates the controller was not gracefully
2789 * disconnected. In that case, we won't be able to flush any data while
2790 * removing the namespaces' disks; fail all the queues now to avoid
2791 * potentially having to clean up the failed sync later.
2792 */
2793 if (ctrl->state == NVME_CTRL_DEAD)
2794 nvme_kill_queues(ctrl);
2795
2796 list_for_each_entry_safe(ns, next, &ctrl->namespaces, list)
2797 nvme_ns_remove(ns);
2798 }
2799 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
2800
2801 static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
2802 {
2803 char *envp[2] = { NULL, NULL };
2804 u32 aen_result = ctrl->aen_result;
2805
2806 ctrl->aen_result = 0;
2807 if (!aen_result)
2808 return;
2809
2810 envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
2811 if (!envp[0])
2812 return;
2813 kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
2814 kfree(envp[0]);
2815 }
2816
2817 static void nvme_async_event_work(struct work_struct *work)
2818 {
2819 struct nvme_ctrl *ctrl =
2820 container_of(work, struct nvme_ctrl, async_event_work);
2821
2822 nvme_aen_uevent(ctrl);
2823 ctrl->ops->submit_async_event(ctrl);
2824 }
2825
2826 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
2827 {
2828
2829 u32 csts;
2830
2831 if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
2832 return false;
2833
2834 if (csts == ~0)
2835 return false;
2836
2837 return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
2838 }
2839
2840 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
2841 {
2842 struct nvme_fw_slot_info_log *log;
2843
2844 log = kmalloc(sizeof(*log), GFP_KERNEL);
2845 if (!log)
2846 return;
2847
2848 if (nvme_get_log(ctrl, NVME_LOG_FW_SLOT, log, sizeof(*log)))
2849 dev_warn(ctrl->device,
2850 "Get FW SLOT INFO log error\n");
2851 kfree(log);
2852 }
2853
2854 static void nvme_fw_act_work(struct work_struct *work)
2855 {
2856 struct nvme_ctrl *ctrl = container_of(work,
2857 struct nvme_ctrl, fw_act_work);
2858 unsigned long fw_act_timeout;
2859
2860 if (ctrl->mtfa)
2861 fw_act_timeout = jiffies +
2862 msecs_to_jiffies(ctrl->mtfa * 100);
2863 else
2864 fw_act_timeout = jiffies +
2865 msecs_to_jiffies(admin_timeout * 1000);
2866
2867 nvme_stop_queues(ctrl);
2868 while (nvme_ctrl_pp_status(ctrl)) {
2869 if (time_after(jiffies, fw_act_timeout)) {
2870 dev_warn(ctrl->device,
2871 "Fw activation timeout, reset controller\n");
2872 nvme_reset_ctrl(ctrl);
2873 break;
2874 }
2875 msleep(100);
2876 }
2877
2878 if (ctrl->state != NVME_CTRL_LIVE)
2879 return;
2880
2881 nvme_start_queues(ctrl);
2882 /* read FW slot information to clear the AER */
2883 nvme_get_fw_slot_info(ctrl);
2884 }
2885
2886 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
2887 union nvme_result *res)
2888 {
2889 u32 result = le32_to_cpu(res->u32);
2890
2891 if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
2892 return;
2893
2894 switch (result & 0x7) {
2895 case NVME_AER_ERROR:
2896 case NVME_AER_SMART:
2897 case NVME_AER_CSS:
2898 case NVME_AER_VS:
2899 ctrl->aen_result = result;
2900 break;
2901 default:
2902 break;
2903 }
2904
2905 switch (result & 0xff07) {
2906 case NVME_AER_NOTICE_NS_CHANGED:
2907 dev_info(ctrl->device, "rescanning\n");
2908 nvme_queue_scan(ctrl);
2909 break;
2910 case NVME_AER_NOTICE_FW_ACT_STARTING:
2911 queue_work(nvme_wq, &ctrl->fw_act_work);
2912 break;
2913 default:
2914 dev_warn(ctrl->device, "async event result %08x\n", result);
2915 }
2916 queue_work(nvme_wq, &ctrl->async_event_work);
2917 }
2918 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
2919
2920 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
2921 {
2922 nvme_stop_keep_alive(ctrl);
2923 flush_work(&ctrl->async_event_work);
2924 flush_work(&ctrl->scan_work);
2925 cancel_work_sync(&ctrl->fw_act_work);
2926 }
2927 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
2928
2929 void nvme_start_ctrl(struct nvme_ctrl *ctrl)
2930 {
2931 if (ctrl->kato)
2932 nvme_start_keep_alive(ctrl);
2933
2934 if (ctrl->queue_count > 1) {
2935 nvme_queue_scan(ctrl);
2936 queue_work(nvme_wq, &ctrl->async_event_work);
2937 nvme_start_queues(ctrl);
2938 }
2939 }
2940 EXPORT_SYMBOL_GPL(nvme_start_ctrl);
2941
2942 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
2943 {
2944 cdev_device_del(&ctrl->cdev, ctrl->device);
2945 }
2946 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
2947
2948 static void nvme_free_ctrl(struct device *dev)
2949 {
2950 struct nvme_ctrl *ctrl =
2951 container_of(dev, struct nvme_ctrl, ctrl_device);
2952 struct nvme_subsystem *subsys = ctrl->subsys;
2953
2954 ida_simple_remove(&nvme_instance_ida, ctrl->instance);
2955 ida_destroy(&ctrl->ns_ida);
2956 kfree(ctrl->effects);
2957
2958 if (subsys) {
2959 mutex_lock(&subsys->lock);
2960 list_del(&ctrl->subsys_entry);
2961 mutex_unlock(&subsys->lock);
2962 sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
2963 }
2964
2965 ctrl->ops->free_ctrl(ctrl);
2966
2967 if (subsys)
2968 nvme_put_subsystem(subsys);
2969 }
2970
2971 /*
2972 * Initialize a NVMe controller structures. This needs to be called during
2973 * earliest initialization so that we have the initialized structured around
2974 * during probing.
2975 */
2976 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
2977 const struct nvme_ctrl_ops *ops, unsigned long quirks)
2978 {
2979 int ret;
2980
2981 ctrl->state = NVME_CTRL_NEW;
2982 spin_lock_init(&ctrl->lock);
2983 INIT_LIST_HEAD(&ctrl->namespaces);
2984 mutex_init(&ctrl->namespaces_mutex);
2985 ctrl->dev = dev;
2986 ctrl->ops = ops;
2987 ctrl->quirks = quirks;
2988 INIT_WORK(&ctrl->scan_work, nvme_scan_work);
2989 INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
2990 INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
2991 INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
2992
2993 ret = ida_simple_get(&nvme_instance_ida, 0, 0, GFP_KERNEL);
2994 if (ret < 0)
2995 goto out;
2996 ctrl->instance = ret;
2997
2998 device_initialize(&ctrl->ctrl_device);
2999 ctrl->device = &ctrl->ctrl_device;
3000 ctrl->device->devt = MKDEV(MAJOR(nvme_chr_devt), ctrl->instance);
3001 ctrl->device->class = nvme_class;
3002 ctrl->device->parent = ctrl->dev;
3003 ctrl->device->groups = nvme_dev_attr_groups;
3004 ctrl->device->release = nvme_free_ctrl;
3005 dev_set_drvdata(ctrl->device, ctrl);
3006 ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
3007 if (ret)
3008 goto out_release_instance;
3009
3010 cdev_init(&ctrl->cdev, &nvme_dev_fops);
3011 ctrl->cdev.owner = ops->module;
3012 ret = cdev_device_add(&ctrl->cdev, ctrl->device);
3013 if (ret)
3014 goto out_free_name;
3015
3016 ida_init(&ctrl->ns_ida);
3017
3018 /*
3019 * Initialize latency tolerance controls. The sysfs files won't
3020 * be visible to userspace unless the device actually supports APST.
3021 */
3022 ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
3023 dev_pm_qos_update_user_latency_tolerance(ctrl->device,
3024 min(default_ps_max_latency_us, (unsigned long)S32_MAX));
3025
3026 return 0;
3027 out_free_name:
3028 kfree_const(dev->kobj.name);
3029 out_release_instance:
3030 ida_simple_remove(&nvme_instance_ida, ctrl->instance);
3031 out:
3032 return ret;
3033 }
3034 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
3035
3036 /**
3037 * nvme_kill_queues(): Ends all namespace queues
3038 * @ctrl: the dead controller that needs to end
3039 *
3040 * Call this function when the driver determines it is unable to get the
3041 * controller in a state capable of servicing IO.
3042 */
3043 void nvme_kill_queues(struct nvme_ctrl *ctrl)
3044 {
3045 struct nvme_ns *ns;
3046
3047 mutex_lock(&ctrl->namespaces_mutex);
3048
3049 /* Forcibly unquiesce queues to avoid blocking dispatch */
3050 if (ctrl->admin_q)
3051 blk_mq_unquiesce_queue(ctrl->admin_q);
3052
3053 list_for_each_entry(ns, &ctrl->namespaces, list) {
3054 /*
3055 * Revalidating a dead namespace sets capacity to 0. This will
3056 * end buffered writers dirtying pages that can't be synced.
3057 */
3058 if (!ns->disk || test_and_set_bit(NVME_NS_DEAD, &ns->flags))
3059 continue;
3060 revalidate_disk(ns->disk);
3061 blk_set_queue_dying(ns->queue);
3062
3063 /* Forcibly unquiesce queues to avoid blocking dispatch */
3064 blk_mq_unquiesce_queue(ns->queue);
3065 }
3066 mutex_unlock(&ctrl->namespaces_mutex);
3067 }
3068 EXPORT_SYMBOL_GPL(nvme_kill_queues);
3069
3070 void nvme_unfreeze(struct nvme_ctrl *ctrl)
3071 {
3072 struct nvme_ns *ns;
3073
3074 mutex_lock(&ctrl->namespaces_mutex);
3075 list_for_each_entry(ns, &ctrl->namespaces, list)
3076 blk_mq_unfreeze_queue(ns->queue);
3077 mutex_unlock(&ctrl->namespaces_mutex);
3078 }
3079 EXPORT_SYMBOL_GPL(nvme_unfreeze);
3080
3081 void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
3082 {
3083 struct nvme_ns *ns;
3084
3085 mutex_lock(&ctrl->namespaces_mutex);
3086 list_for_each_entry(ns, &ctrl->namespaces, list) {
3087 timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
3088 if (timeout <= 0)
3089 break;
3090 }
3091 mutex_unlock(&ctrl->namespaces_mutex);
3092 }
3093 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
3094
3095 void nvme_wait_freeze(struct nvme_ctrl *ctrl)
3096 {
3097 struct nvme_ns *ns;
3098
3099 mutex_lock(&ctrl->namespaces_mutex);
3100 list_for_each_entry(ns, &ctrl->namespaces, list)
3101 blk_mq_freeze_queue_wait(ns->queue);
3102 mutex_unlock(&ctrl->namespaces_mutex);
3103 }
3104 EXPORT_SYMBOL_GPL(nvme_wait_freeze);
3105
3106 void nvme_start_freeze(struct nvme_ctrl *ctrl)
3107 {
3108 struct nvme_ns *ns;
3109
3110 mutex_lock(&ctrl->namespaces_mutex);
3111 list_for_each_entry(ns, &ctrl->namespaces, list)
3112 blk_freeze_queue_start(ns->queue);
3113 mutex_unlock(&ctrl->namespaces_mutex);
3114 }
3115 EXPORT_SYMBOL_GPL(nvme_start_freeze);
3116
3117 void nvme_stop_queues(struct nvme_ctrl *ctrl)
3118 {
3119 struct nvme_ns *ns;
3120
3121 mutex_lock(&ctrl->namespaces_mutex);
3122 list_for_each_entry(ns, &ctrl->namespaces, list)
3123 blk_mq_quiesce_queue(ns->queue);
3124 mutex_unlock(&ctrl->namespaces_mutex);
3125 }
3126 EXPORT_SYMBOL_GPL(nvme_stop_queues);
3127
3128 void nvme_start_queues(struct nvme_ctrl *ctrl)
3129 {
3130 struct nvme_ns *ns;
3131
3132 mutex_lock(&ctrl->namespaces_mutex);
3133 list_for_each_entry(ns, &ctrl->namespaces, list)
3134 blk_mq_unquiesce_queue(ns->queue);
3135 mutex_unlock(&ctrl->namespaces_mutex);
3136 }
3137 EXPORT_SYMBOL_GPL(nvme_start_queues);
3138
3139 int nvme_reinit_tagset(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set)
3140 {
3141 if (!ctrl->ops->reinit_request)
3142 return 0;
3143
3144 return blk_mq_tagset_iter(set, set->driver_data,
3145 ctrl->ops->reinit_request);
3146 }
3147 EXPORT_SYMBOL_GPL(nvme_reinit_tagset);
3148
3149 int __init nvme_core_init(void)
3150 {
3151 int result;
3152
3153 nvme_wq = alloc_workqueue("nvme-wq",
3154 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
3155 if (!nvme_wq)
3156 return -ENOMEM;
3157
3158 result = alloc_chrdev_region(&nvme_chr_devt, 0, NVME_MINORS, "nvme");
3159 if (result < 0)
3160 goto destroy_wq;
3161
3162 nvme_class = class_create(THIS_MODULE, "nvme");
3163 if (IS_ERR(nvme_class)) {
3164 result = PTR_ERR(nvme_class);
3165 goto unregister_chrdev;
3166 }
3167
3168 nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
3169 if (IS_ERR(nvme_subsys_class)) {
3170 result = PTR_ERR(nvme_subsys_class);
3171 goto destroy_class;
3172 }
3173 return 0;
3174
3175 destroy_class:
3176 class_destroy(nvme_class);
3177 unregister_chrdev:
3178 unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
3179 destroy_wq:
3180 destroy_workqueue(nvme_wq);
3181 return result;
3182 }
3183
3184 void nvme_core_exit(void)
3185 {
3186 ida_destroy(&nvme_subsystems_ida);
3187 class_destroy(nvme_subsys_class);
3188 class_destroy(nvme_class);
3189 unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
3190 destroy_workqueue(nvme_wq);
3191 }
3192
3193 MODULE_LICENSE("GPL");
3194 MODULE_VERSION("1.0");
3195 module_init(nvme_core_init);
3196 module_exit(nvme_core_exit);
ubuntu-bionic-kernel mirror