4 * Copyright (c) Intel Corporation.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include "nvme_internal.h"
38 static int nvme_ctrlr_construct_and_submit_aer(struct spdk_nvme_ctrlr
*ctrlr
,
39 struct nvme_async_event_request
*aer
);
42 nvme_ctrlr_get_cc(struct spdk_nvme_ctrlr
*ctrlr
, union spdk_nvme_cc_register
*cc
)
44 return nvme_transport_ctrlr_get_reg_4(ctrlr
, offsetof(struct spdk_nvme_registers
, cc
.raw
),
49 nvme_ctrlr_get_csts(struct spdk_nvme_ctrlr
*ctrlr
, union spdk_nvme_csts_register
*csts
)
51 return nvme_transport_ctrlr_get_reg_4(ctrlr
, offsetof(struct spdk_nvme_registers
, csts
.raw
),
56 nvme_ctrlr_get_cap(struct spdk_nvme_ctrlr
*ctrlr
, union spdk_nvme_cap_register
*cap
)
58 return nvme_transport_ctrlr_get_reg_8(ctrlr
, offsetof(struct spdk_nvme_registers
, cap
.raw
),
63 nvme_ctrlr_get_vs(struct spdk_nvme_ctrlr
*ctrlr
, union spdk_nvme_vs_register
*vs
)
65 return nvme_transport_ctrlr_get_reg_4(ctrlr
, offsetof(struct spdk_nvme_registers
, vs
.raw
),
70 nvme_ctrlr_set_cc(struct spdk_nvme_ctrlr
*ctrlr
, const union spdk_nvme_cc_register
*cc
)
72 return nvme_transport_ctrlr_set_reg_4(ctrlr
, offsetof(struct spdk_nvme_registers
, cc
.raw
),
77 spdk_nvme_ctrlr_opts_set_defaults(struct spdk_nvme_ctrlr_opts
*opts
)
79 opts
->num_io_queues
= DEFAULT_MAX_IO_QUEUES
;
80 opts
->use_cmb_sqs
= false;
81 opts
->arb_mechanism
= SPDK_NVME_CC_AMS_RR
;
82 opts
->keep_alive_timeout_ms
= 10 * 1000;
83 opts
->io_queue_size
= DEFAULT_IO_QUEUE_SIZE
;
84 strncpy(opts
->hostnqn
, DEFAULT_HOSTNQN
, sizeof(opts
->hostnqn
));
85 opts
->io_queue_requests
= DEFAULT_IO_QUEUE_REQUESTS
;
89 * This function will be called when the process allocates the IO qpair.
90 * Note: the ctrlr_lock must be held when calling this function.
93 nvme_ctrlr_proc_add_io_qpair(struct spdk_nvme_qpair
*qpair
)
95 struct spdk_nvme_ctrlr_process
*active_proc
;
96 struct spdk_nvme_ctrlr
*ctrlr
= qpair
->ctrlr
;
99 TAILQ_FOREACH(active_proc
, &ctrlr
->active_procs
, tailq
) {
100 if (active_proc
->pid
== pid
) {
101 TAILQ_INSERT_TAIL(&active_proc
->allocated_io_qpairs
, qpair
,
109 * This function will be called when the process frees the IO qpair.
110 * Note: the ctrlr_lock must be held when calling this function.
113 nvme_ctrlr_proc_remove_io_qpair(struct spdk_nvme_qpair
*qpair
)
115 struct spdk_nvme_ctrlr_process
*active_proc
;
116 struct spdk_nvme_ctrlr
*ctrlr
= qpair
->ctrlr
;
117 struct spdk_nvme_qpair
*active_qpair
, *tmp_qpair
;
118 pid_t pid
= getpid();
119 bool proc_found
= false;
121 TAILQ_FOREACH(active_proc
, &ctrlr
->active_procs
, tailq
) {
122 if (active_proc
->pid
== pid
) {
128 if (proc_found
== false) {
132 TAILQ_FOREACH_SAFE(active_qpair
, &active_proc
->allocated_io_qpairs
,
133 per_process_tailq
, tmp_qpair
) {
134 if (active_qpair
== qpair
) {
135 TAILQ_REMOVE(&active_proc
->allocated_io_qpairs
,
136 active_qpair
, per_process_tailq
);
143 struct spdk_nvme_qpair
*
144 spdk_nvme_ctrlr_alloc_io_qpair(struct spdk_nvme_ctrlr
*ctrlr
,
145 enum spdk_nvme_qprio qprio
)
148 struct spdk_nvme_qpair
*qpair
;
149 union spdk_nvme_cc_register cc
;
151 if (nvme_ctrlr_get_cc(ctrlr
, &cc
)) {
152 SPDK_ERRLOG("get_cc failed\n");
156 /* Only the low 2 bits (values 0, 1, 2, 3) of QPRIO are valid. */
157 if ((qprio
& 3) != qprio
) {
162 * Only value SPDK_NVME_QPRIO_URGENT(0) is valid for the
163 * default round robin arbitration method.
165 if ((cc
.bits
.ams
== SPDK_NVME_CC_AMS_RR
) && (qprio
!= SPDK_NVME_QPRIO_URGENT
)) {
166 SPDK_ERRLOG("invalid queue priority for default round robin arbitration method\n");
170 nvme_robust_mutex_lock(&ctrlr
->ctrlr_lock
);
173 * Get the first available I/O queue ID.
175 qid
= spdk_bit_array_find_first_set(ctrlr
->free_io_qids
, 1);
176 if (qid
> ctrlr
->opts
.num_io_queues
) {
177 SPDK_ERRLOG("No free I/O queue IDs\n");
178 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
182 qpair
= nvme_transport_ctrlr_create_io_qpair(ctrlr
, qid
, qprio
);
184 SPDK_ERRLOG("transport->ctrlr_create_io_qpair() failed\n");
185 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
188 spdk_bit_array_clear(ctrlr
->free_io_qids
, qid
);
189 TAILQ_INSERT_TAIL(&ctrlr
->active_io_qpairs
, qpair
, tailq
);
191 nvme_ctrlr_proc_add_io_qpair(qpair
);
193 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
199 spdk_nvme_ctrlr_free_io_qpair(struct spdk_nvme_qpair
*qpair
)
201 struct spdk_nvme_ctrlr
*ctrlr
;
207 ctrlr
= qpair
->ctrlr
;
209 if (qpair
->in_completion_context
) {
211 * There are many cases where it is convenient to delete an io qpair in the context
212 * of that qpair's completion routine. To handle this properly, set a flag here
213 * so that the completion routine will perform an actual delete after the context
216 qpair
->delete_after_completion_context
= 1;
220 nvme_robust_mutex_lock(&ctrlr
->ctrlr_lock
);
222 nvme_ctrlr_proc_remove_io_qpair(qpair
);
224 TAILQ_REMOVE(&ctrlr
->active_io_qpairs
, qpair
, tailq
);
225 spdk_bit_array_set(ctrlr
->free_io_qids
, qpair
->id
);
227 spdk_free(qpair
->req_buf
);
229 if (nvme_transport_ctrlr_delete_io_qpair(ctrlr
, qpair
)) {
230 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
234 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
239 nvme_ctrlr_construct_intel_support_log_page_list(struct spdk_nvme_ctrlr
*ctrlr
,
240 struct spdk_nvme_intel_log_page_directory
*log_page_directory
)
242 if (log_page_directory
== NULL
) {
246 if (ctrlr
->cdata
.vid
!= SPDK_PCI_VID_INTEL
) {
250 ctrlr
->log_page_supported
[SPDK_NVME_INTEL_LOG_PAGE_DIRECTORY
] = true;
252 if (log_page_directory
->read_latency_log_len
||
253 (ctrlr
->quirks
& NVME_INTEL_QUIRK_READ_LATENCY
)) {
254 ctrlr
->log_page_supported
[SPDK_NVME_INTEL_LOG_READ_CMD_LATENCY
] = true;
256 if (log_page_directory
->write_latency_log_len
||
257 (ctrlr
->quirks
& NVME_INTEL_QUIRK_WRITE_LATENCY
)) {
258 ctrlr
->log_page_supported
[SPDK_NVME_INTEL_LOG_WRITE_CMD_LATENCY
] = true;
260 if (log_page_directory
->temperature_statistics_log_len
) {
261 ctrlr
->log_page_supported
[SPDK_NVME_INTEL_LOG_TEMPERATURE
] = true;
263 if (log_page_directory
->smart_log_len
) {
264 ctrlr
->log_page_supported
[SPDK_NVME_INTEL_LOG_SMART
] = true;
266 if (log_page_directory
->marketing_description_log_len
) {
267 ctrlr
->log_page_supported
[SPDK_NVME_INTEL_MARKETING_DESCRIPTION
] = true;
271 static int nvme_ctrlr_set_intel_support_log_pages(struct spdk_nvme_ctrlr
*ctrlr
)
273 uint64_t phys_addr
= 0;
274 struct nvme_completion_poll_status status
;
275 struct spdk_nvme_intel_log_page_directory
*log_page_directory
;
277 log_page_directory
= spdk_zmalloc(sizeof(struct spdk_nvme_intel_log_page_directory
),
279 if (log_page_directory
== NULL
) {
280 SPDK_ERRLOG("could not allocate log_page_directory\n");
285 spdk_nvme_ctrlr_cmd_get_log_page(ctrlr
, SPDK_NVME_INTEL_LOG_PAGE_DIRECTORY
, SPDK_NVME_GLOBAL_NS_TAG
,
286 log_page_directory
, sizeof(struct spdk_nvme_intel_log_page_directory
), 0,
287 nvme_completion_poll_cb
,
289 while (status
.done
== false) {
290 spdk_nvme_qpair_process_completions(ctrlr
->adminq
, 0);
292 if (spdk_nvme_cpl_is_error(&status
.cpl
)) {
293 spdk_free(log_page_directory
);
294 SPDK_ERRLOG("nvme_ctrlr_cmd_get_log_page failed!\n");
298 nvme_ctrlr_construct_intel_support_log_page_list(ctrlr
, log_page_directory
);
299 spdk_free(log_page_directory
);
304 nvme_ctrlr_set_supported_log_pages(struct spdk_nvme_ctrlr
*ctrlr
)
306 memset(ctrlr
->log_page_supported
, 0, sizeof(ctrlr
->log_page_supported
));
307 /* Mandatory pages */
308 ctrlr
->log_page_supported
[SPDK_NVME_LOG_ERROR
] = true;
309 ctrlr
->log_page_supported
[SPDK_NVME_LOG_HEALTH_INFORMATION
] = true;
310 ctrlr
->log_page_supported
[SPDK_NVME_LOG_FIRMWARE_SLOT
] = true;
311 if (ctrlr
->cdata
.lpa
.celp
) {
312 ctrlr
->log_page_supported
[SPDK_NVME_LOG_COMMAND_EFFECTS_LOG
] = true;
314 if (ctrlr
->cdata
.vid
== SPDK_PCI_VID_INTEL
) {
315 nvme_ctrlr_set_intel_support_log_pages(ctrlr
);
320 nvme_ctrlr_set_intel_supported_features(struct spdk_nvme_ctrlr
*ctrlr
)
322 ctrlr
->feature_supported
[SPDK_NVME_INTEL_FEAT_MAX_LBA
] = true;
323 ctrlr
->feature_supported
[SPDK_NVME_INTEL_FEAT_NATIVE_MAX_LBA
] = true;
324 ctrlr
->feature_supported
[SPDK_NVME_INTEL_FEAT_POWER_GOVERNOR_SETTING
] = true;
325 ctrlr
->feature_supported
[SPDK_NVME_INTEL_FEAT_SMBUS_ADDRESS
] = true;
326 ctrlr
->feature_supported
[SPDK_NVME_INTEL_FEAT_LED_PATTERN
] = true;
327 ctrlr
->feature_supported
[SPDK_NVME_INTEL_FEAT_RESET_TIMED_WORKLOAD_COUNTERS
] = true;
328 ctrlr
->feature_supported
[SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING
] = true;
332 nvme_ctrlr_set_supported_features(struct spdk_nvme_ctrlr
*ctrlr
)
334 memset(ctrlr
->feature_supported
, 0, sizeof(ctrlr
->feature_supported
));
335 /* Mandatory features */
336 ctrlr
->feature_supported
[SPDK_NVME_FEAT_ARBITRATION
] = true;
337 ctrlr
->feature_supported
[SPDK_NVME_FEAT_POWER_MANAGEMENT
] = true;
338 ctrlr
->feature_supported
[SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD
] = true;
339 ctrlr
->feature_supported
[SPDK_NVME_FEAT_ERROR_RECOVERY
] = true;
340 ctrlr
->feature_supported
[SPDK_NVME_FEAT_NUMBER_OF_QUEUES
] = true;
341 ctrlr
->feature_supported
[SPDK_NVME_FEAT_INTERRUPT_COALESCING
] = true;
342 ctrlr
->feature_supported
[SPDK_NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION
] = true;
343 ctrlr
->feature_supported
[SPDK_NVME_FEAT_WRITE_ATOMICITY
] = true;
344 ctrlr
->feature_supported
[SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION
] = true;
345 /* Optional features */
346 if (ctrlr
->cdata
.vwc
.present
) {
347 ctrlr
->feature_supported
[SPDK_NVME_FEAT_VOLATILE_WRITE_CACHE
] = true;
349 if (ctrlr
->cdata
.apsta
.supported
) {
350 ctrlr
->feature_supported
[SPDK_NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION
] = true;
352 if (ctrlr
->cdata
.hmpre
) {
353 ctrlr
->feature_supported
[SPDK_NVME_FEAT_HOST_MEM_BUFFER
] = true;
355 if (ctrlr
->cdata
.vid
== SPDK_PCI_VID_INTEL
) {
356 nvme_ctrlr_set_intel_supported_features(ctrlr
);
361 nvme_ctrlr_fail(struct spdk_nvme_ctrlr
*ctrlr
, bool hot_remove
)
364 * Set the flag here and leave the work failure of qpairs to
365 * spdk_nvme_qpair_process_completions().
368 ctrlr
->is_removed
= true;
370 ctrlr
->is_failed
= true;
374 nvme_ctrlr_shutdown(struct spdk_nvme_ctrlr
*ctrlr
)
376 union spdk_nvme_cc_register cc
;
377 union spdk_nvme_csts_register csts
;
380 if (ctrlr
->is_removed
) {
384 if (nvme_ctrlr_get_cc(ctrlr
, &cc
)) {
385 SPDK_ERRLOG("get_cc() failed\n");
389 cc
.bits
.shn
= SPDK_NVME_SHN_NORMAL
;
391 if (nvme_ctrlr_set_cc(ctrlr
, &cc
)) {
392 SPDK_ERRLOG("set_cc() failed\n");
397 * The NVMe spec does not define a timeout period
398 * for shutdown notification, so we just pick
399 * 5 seconds as a reasonable amount of time to
400 * wait before proceeding.
403 if (nvme_ctrlr_get_csts(ctrlr
, &csts
)) {
404 SPDK_ERRLOG("get_csts() failed\n");
408 if (csts
.bits
.shst
== SPDK_NVME_SHST_COMPLETE
) {
409 SPDK_TRACELOG(SPDK_TRACE_NVME
, "shutdown complete\n");
415 } while (ms_waited
< 5000);
417 SPDK_ERRLOG("did not shutdown within 5 seconds\n");
421 nvme_ctrlr_enable(struct spdk_nvme_ctrlr
*ctrlr
)
423 union spdk_nvme_cc_register cc
;
426 rc
= nvme_transport_ctrlr_enable(ctrlr
);
428 SPDK_ERRLOG("transport ctrlr_enable failed\n");
432 if (nvme_ctrlr_get_cc(ctrlr
, &cc
)) {
433 SPDK_ERRLOG("get_cc() failed\n");
437 if (cc
.bits
.en
!= 0) {
438 SPDK_ERRLOG("%s called with CC.EN = 1\n", __func__
);
445 cc
.bits
.iosqes
= 6; /* SQ entry size == 64 == 2^6 */
446 cc
.bits
.iocqes
= 4; /* CQ entry size == 16 == 2^4 */
448 /* Page size is 2 ^ (12 + mps). */
449 cc
.bits
.mps
= spdk_u32log2(PAGE_SIZE
) - 12;
451 switch (ctrlr
->opts
.arb_mechanism
) {
452 case SPDK_NVME_CC_AMS_RR
:
454 case SPDK_NVME_CC_AMS_WRR
:
455 if (SPDK_NVME_CAP_AMS_WRR
& ctrlr
->cap
.bits
.ams
) {
459 case SPDK_NVME_CC_AMS_VS
:
460 if (SPDK_NVME_CAP_AMS_VS
& ctrlr
->cap
.bits
.ams
) {
468 cc
.bits
.ams
= ctrlr
->opts
.arb_mechanism
;
470 if (nvme_ctrlr_set_cc(ctrlr
, &cc
)) {
471 SPDK_ERRLOG("set_cc() failed\n");
480 nvme_ctrlr_state_string(enum nvme_ctrlr_state state
)
483 case NVME_CTRLR_STATE_INIT
:
485 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1
:
486 return "disable and wait for CSTS.RDY = 1";
487 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0
:
488 return "disable and wait for CSTS.RDY = 0";
489 case NVME_CTRLR_STATE_ENABLE
:
490 return "enable controller by writing CC.EN = 1";
491 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1
:
492 return "wait for CSTS.RDY = 1";
493 case NVME_CTRLR_STATE_READY
:
501 nvme_ctrlr_set_state(struct spdk_nvme_ctrlr
*ctrlr
, enum nvme_ctrlr_state state
,
502 uint64_t timeout_in_ms
)
504 ctrlr
->state
= state
;
505 if (timeout_in_ms
== NVME_TIMEOUT_INFINITE
) {
506 SPDK_TRACELOG(SPDK_TRACE_NVME
, "setting state to %s (no timeout)\n",
507 nvme_ctrlr_state_string(ctrlr
->state
));
508 ctrlr
->state_timeout_tsc
= NVME_TIMEOUT_INFINITE
;
510 SPDK_TRACELOG(SPDK_TRACE_NVME
, "setting state to %s (timeout %" PRIu64
" ms)\n",
511 nvme_ctrlr_state_string(ctrlr
->state
), timeout_in_ms
);
512 ctrlr
->state_timeout_tsc
= spdk_get_ticks() + (timeout_in_ms
* spdk_get_ticks_hz()) / 1000;
517 spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr
*ctrlr
)
520 struct spdk_nvme_qpair
*qpair
;
521 struct nvme_request
*req
, *tmp
;
523 nvme_robust_mutex_lock(&ctrlr
->ctrlr_lock
);
525 if (ctrlr
->is_resetting
|| ctrlr
->is_failed
) {
527 * Controller is already resetting or has failed. Return
528 * immediately since there is no need to kick off another
529 * reset in these cases.
531 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
535 ctrlr
->is_resetting
= true;
537 SPDK_NOTICELOG("resetting controller\n");
539 /* Free all of the queued abort requests */
540 STAILQ_FOREACH_SAFE(req
, &ctrlr
->queued_aborts
, stailq
, tmp
) {
541 STAILQ_REMOVE_HEAD(&ctrlr
->queued_aborts
, stailq
);
542 nvme_free_request(req
);
543 ctrlr
->outstanding_aborts
--;
546 /* Disable all queues before disabling the controller hardware. */
547 nvme_qpair_disable(ctrlr
->adminq
);
548 TAILQ_FOREACH(qpair
, &ctrlr
->active_io_qpairs
, tailq
) {
549 nvme_qpair_disable(qpair
);
552 /* Set the state back to INIT to cause a full hardware reset. */
553 nvme_ctrlr_set_state(ctrlr
, NVME_CTRLR_STATE_INIT
, NVME_TIMEOUT_INFINITE
);
555 while (ctrlr
->state
!= NVME_CTRLR_STATE_READY
) {
556 if (nvme_ctrlr_process_init(ctrlr
) != 0) {
557 SPDK_ERRLOG("%s: controller reinitialization failed\n", __func__
);
558 nvme_ctrlr_fail(ctrlr
, false);
564 if (!ctrlr
->is_failed
) {
565 /* Reinitialize qpairs */
566 TAILQ_FOREACH(qpair
, &ctrlr
->active_io_qpairs
, tailq
) {
567 if (nvme_transport_ctrlr_reinit_io_qpair(ctrlr
, qpair
) != 0) {
568 nvme_ctrlr_fail(ctrlr
, false);
574 ctrlr
->is_resetting
= false;
576 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
582 nvme_ctrlr_identify(struct spdk_nvme_ctrlr
*ctrlr
)
584 struct nvme_completion_poll_status status
;
588 rc
= nvme_ctrlr_cmd_identify_controller(ctrlr
, &ctrlr
->cdata
,
589 nvme_completion_poll_cb
, &status
);
594 while (status
.done
== false) {
595 spdk_nvme_qpair_process_completions(ctrlr
->adminq
, 0);
597 if (spdk_nvme_cpl_is_error(&status
.cpl
)) {
598 SPDK_ERRLOG("nvme_identify_controller failed!\n");
603 * Use MDTS to ensure our default max_xfer_size doesn't exceed what the
604 * controller supports.
606 ctrlr
->max_xfer_size
= nvme_transport_ctrlr_get_max_xfer_size(ctrlr
);
607 SPDK_TRACELOG(SPDK_TRACE_NVME
, "transport max_xfer_size %u\n", ctrlr
->max_xfer_size
);
608 if (ctrlr
->cdata
.mdts
> 0) {
609 ctrlr
->max_xfer_size
= spdk_min(ctrlr
->max_xfer_size
,
610 ctrlr
->min_page_size
* (1 << (ctrlr
->cdata
.mdts
)));
611 SPDK_TRACELOG(SPDK_TRACE_NVME
, "MDTS max_xfer_size %u\n", ctrlr
->max_xfer_size
);
618 nvme_ctrlr_set_num_qpairs(struct spdk_nvme_ctrlr
*ctrlr
)
620 struct nvme_completion_poll_status status
;
621 int cq_allocated
, sq_allocated
;
627 if (ctrlr
->opts
.num_io_queues
> SPDK_NVME_MAX_IO_QUEUES
) {
628 SPDK_NOTICELOG("Limiting requested num_io_queues %u to max %d\n",
629 ctrlr
->opts
.num_io_queues
, SPDK_NVME_MAX_IO_QUEUES
);
630 ctrlr
->opts
.num_io_queues
= SPDK_NVME_MAX_IO_QUEUES
;
631 } else if (ctrlr
->opts
.num_io_queues
< 1) {
632 SPDK_NOTICELOG("Requested num_io_queues 0, increasing to 1\n");
633 ctrlr
->opts
.num_io_queues
= 1;
636 rc
= nvme_ctrlr_cmd_set_num_queues(ctrlr
, ctrlr
->opts
.num_io_queues
,
637 nvme_completion_poll_cb
, &status
);
642 while (status
.done
== false) {
643 spdk_nvme_qpair_process_completions(ctrlr
->adminq
, 0);
645 if (spdk_nvme_cpl_is_error(&status
.cpl
)) {
646 SPDK_ERRLOG("nvme_set_num_queues failed!\n");
651 * Data in cdw0 is 0-based.
652 * Lower 16-bits indicate number of submission queues allocated.
653 * Upper 16-bits indicate number of completion queues allocated.
655 sq_allocated
= (status
.cpl
.cdw0
& 0xFFFF) + 1;
656 cq_allocated
= (status
.cpl
.cdw0
>> 16) + 1;
658 ctrlr
->opts
.num_io_queues
= spdk_min(sq_allocated
, cq_allocated
);
660 ctrlr
->free_io_qids
= spdk_bit_array_create(ctrlr
->opts
.num_io_queues
+ 1);
661 if (ctrlr
->free_io_qids
== NULL
) {
665 /* Initialize list of free I/O queue IDs. QID 0 is the admin queue. */
666 spdk_bit_array_clear(ctrlr
->free_io_qids
, 0);
667 for (i
= 1; i
<= ctrlr
->opts
.num_io_queues
; i
++) {
668 spdk_bit_array_set(ctrlr
->free_io_qids
, i
);
675 nvme_ctrlr_set_keep_alive_timeout(struct spdk_nvme_ctrlr
*ctrlr
)
677 struct nvme_completion_poll_status status
;
678 uint32_t keep_alive_interval_ms
;
681 if (ctrlr
->opts
.keep_alive_timeout_ms
== 0) {
685 if (ctrlr
->cdata
.kas
== 0) {
686 SPDK_TRACELOG(SPDK_TRACE_NVME
, "Controller KAS is 0 - not enabling Keep Alive\n");
687 ctrlr
->opts
.keep_alive_timeout_ms
= 0;
691 /* Retrieve actual keep alive timeout, since the controller may have adjusted it. */
693 rc
= spdk_nvme_ctrlr_cmd_get_feature(ctrlr
, SPDK_NVME_FEAT_KEEP_ALIVE_TIMER
, 0, NULL
, 0,
694 nvme_completion_poll_cb
, &status
);
696 SPDK_ERRLOG("Keep alive timeout Get Feature failed: %d\n", rc
);
697 ctrlr
->opts
.keep_alive_timeout_ms
= 0;
701 while (status
.done
== false) {
702 spdk_nvme_qpair_process_completions(ctrlr
->adminq
, 0);
704 if (spdk_nvme_cpl_is_error(&status
.cpl
)) {
705 SPDK_ERRLOG("Keep alive timeout Get Feature failed: SC %x SCT %x\n",
706 status
.cpl
.status
.sc
, status
.cpl
.status
.sct
);
707 ctrlr
->opts
.keep_alive_timeout_ms
= 0;
711 if (ctrlr
->opts
.keep_alive_timeout_ms
!= status
.cpl
.cdw0
) {
712 SPDK_TRACELOG(SPDK_TRACE_NVME
, "Controller adjusted keep alive timeout to %u ms\n",
716 ctrlr
->opts
.keep_alive_timeout_ms
= status
.cpl
.cdw0
;
718 keep_alive_interval_ms
= ctrlr
->opts
.keep_alive_timeout_ms
/ 2;
719 if (keep_alive_interval_ms
== 0) {
720 keep_alive_interval_ms
= 1;
722 SPDK_TRACELOG(SPDK_TRACE_NVME
, "Sending keep alive every %u ms\n", keep_alive_interval_ms
);
724 ctrlr
->keep_alive_interval_ticks
= (keep_alive_interval_ms
* spdk_get_ticks_hz()) / UINT64_C(1000);
726 /* Schedule the first Keep Alive to be sent as soon as possible. */
727 ctrlr
->next_keep_alive_tick
= spdk_get_ticks();
733 nvme_ctrlr_destruct_namespaces(struct spdk_nvme_ctrlr
*ctrlr
)
736 uint32_t i
, num_ns
= ctrlr
->num_ns
;
738 for (i
= 0; i
< num_ns
; i
++) {
739 nvme_ns_destruct(&ctrlr
->ns
[i
]);
742 spdk_free(ctrlr
->ns
);
748 spdk_free(ctrlr
->nsdata
);
749 ctrlr
->nsdata
= NULL
;
754 nvme_ctrlr_construct_namespaces(struct spdk_nvme_ctrlr
*ctrlr
)
756 uint32_t i
, nn
= ctrlr
->cdata
.nn
;
757 uint64_t phys_addr
= 0;
760 SPDK_ERRLOG("controller has 0 namespaces\n");
764 /* ctrlr->num_ns may be 0 (startup) or a different number of namespaces (reset),
765 * so check if we need to reallocate.
767 if (nn
!= ctrlr
->num_ns
) {
768 nvme_ctrlr_destruct_namespaces(ctrlr
);
770 ctrlr
->ns
= spdk_zmalloc(nn
* sizeof(struct spdk_nvme_ns
), 64,
772 if (ctrlr
->ns
== NULL
) {
776 ctrlr
->nsdata
= spdk_zmalloc(nn
* sizeof(struct spdk_nvme_ns_data
), 64,
778 if (ctrlr
->nsdata
== NULL
) {
785 for (i
= 0; i
< nn
; i
++) {
786 struct spdk_nvme_ns
*ns
= &ctrlr
->ns
[i
];
787 uint32_t nsid
= i
+ 1;
789 if (nvme_ns_construct(ns
, nsid
, ctrlr
) != 0) {
797 nvme_ctrlr_destruct_namespaces(ctrlr
);
802 nvme_ctrlr_async_event_cb(void *arg
, const struct spdk_nvme_cpl
*cpl
)
804 struct nvme_async_event_request
*aer
= arg
;
805 struct spdk_nvme_ctrlr
*ctrlr
= aer
->ctrlr
;
807 if (cpl
->status
.sc
== SPDK_NVME_SC_ABORTED_SQ_DELETION
) {
809 * This is simulated when controller is being shut down, to
810 * effectively abort outstanding asynchronous event requests
811 * and make sure all memory is freed. Do not repost the
812 * request in this case.
817 if (ctrlr
->aer_cb_fn
!= NULL
) {
818 ctrlr
->aer_cb_fn(ctrlr
->aer_cb_arg
, cpl
);
822 * Repost another asynchronous event request to replace the one
823 * that just completed.
825 if (nvme_ctrlr_construct_and_submit_aer(ctrlr
, aer
)) {
827 * We can't do anything to recover from a failure here,
828 * so just print a warning message and leave the AER unsubmitted.
830 SPDK_ERRLOG("resubmitting AER failed!\n");
835 nvme_ctrlr_construct_and_submit_aer(struct spdk_nvme_ctrlr
*ctrlr
,
836 struct nvme_async_event_request
*aer
)
838 struct nvme_request
*req
;
841 req
= nvme_allocate_request_null(ctrlr
->adminq
, nvme_ctrlr_async_event_cb
, aer
);
847 req
->cmd
.opc
= SPDK_NVME_OPC_ASYNC_EVENT_REQUEST
;
848 return nvme_ctrlr_submit_admin_request(ctrlr
, req
);
852 nvme_ctrlr_configure_aer(struct spdk_nvme_ctrlr
*ctrlr
)
854 union spdk_nvme_critical_warning_state state
;
855 struct nvme_async_event_request
*aer
;
857 struct nvme_completion_poll_status status
;
863 state
.bits
.reserved
= 0;
864 rc
= nvme_ctrlr_cmd_set_async_event_config(ctrlr
, state
, nvme_completion_poll_cb
, &status
);
869 while (status
.done
== false) {
870 spdk_nvme_qpair_process_completions(ctrlr
->adminq
, 0);
872 if (spdk_nvme_cpl_is_error(&status
.cpl
)) {
873 SPDK_ERRLOG("nvme_ctrlr_cmd_set_async_event_config failed!\n");
877 /* aerl is a zero-based value, so we need to add 1 here. */
878 ctrlr
->num_aers
= spdk_min(NVME_MAX_ASYNC_EVENTS
, (ctrlr
->cdata
.aerl
+ 1));
880 for (i
= 0; i
< ctrlr
->num_aers
; i
++) {
881 aer
= &ctrlr
->aer
[i
];
882 if (nvme_ctrlr_construct_and_submit_aer(ctrlr
, aer
)) {
883 SPDK_ERRLOG("nvme_ctrlr_construct_and_submit_aer failed!\n");
892 * This function will be called when a process is using the controller.
893 * 1. For the primary process, it is called when constructing the controller.
894 * 2. For the secondary process, it is called at probing the controller.
895 * Note: will check whether the process is already added for the same process.
898 nvme_ctrlr_add_process(struct spdk_nvme_ctrlr
*ctrlr
, void *devhandle
)
900 struct spdk_nvme_ctrlr_process
*ctrlr_proc
, *active_proc
;
901 pid_t pid
= getpid();
903 /* Check whether the process is already added or not */
904 TAILQ_FOREACH(active_proc
, &ctrlr
->active_procs
, tailq
) {
905 if (active_proc
->pid
== pid
) {
910 /* Initialize the per process properties for this ctrlr */
911 ctrlr_proc
= spdk_zmalloc(sizeof(struct spdk_nvme_ctrlr_process
), 64, NULL
);
912 if (ctrlr_proc
== NULL
) {
913 SPDK_ERRLOG("failed to allocate memory to track the process props\n");
918 ctrlr_proc
->is_primary
= spdk_process_is_primary();
919 ctrlr_proc
->pid
= pid
;
920 STAILQ_INIT(&ctrlr_proc
->active_reqs
);
921 ctrlr_proc
->devhandle
= devhandle
;
923 TAILQ_INIT(&ctrlr_proc
->allocated_io_qpairs
);
925 TAILQ_INSERT_TAIL(&ctrlr
->active_procs
, ctrlr_proc
, tailq
);
931 * This function will be called when the process detaches the controller.
932 * Note: the ctrlr_lock must be held when calling this function.
935 nvme_ctrlr_remove_process(struct spdk_nvme_ctrlr
*ctrlr
,
936 struct spdk_nvme_ctrlr_process
*proc
)
938 struct spdk_nvme_qpair
*qpair
, *tmp_qpair
;
940 assert(STAILQ_EMPTY(&proc
->active_reqs
));
942 TAILQ_FOREACH_SAFE(qpair
, &proc
->allocated_io_qpairs
, per_process_tailq
, tmp_qpair
) {
943 spdk_nvme_ctrlr_free_io_qpair(qpair
);
946 TAILQ_REMOVE(&ctrlr
->active_procs
, proc
, tailq
);
952 * This function will be called when the process exited unexpectedly
953 * in order to free any incomplete nvme request, allocated IO qpairs
954 * and allocated memory.
955 * Note: the ctrlr_lock must be held when calling this function.
958 nvme_ctrlr_cleanup_process(struct spdk_nvme_ctrlr_process
*proc
)
960 struct nvme_request
*req
, *tmp_req
;
961 struct spdk_nvme_qpair
*qpair
, *tmp_qpair
;
963 STAILQ_FOREACH_SAFE(req
, &proc
->active_reqs
, stailq
, tmp_req
) {
964 STAILQ_REMOVE(&proc
->active_reqs
, req
, nvme_request
, stailq
);
966 assert(req
->pid
== proc
->pid
);
968 nvme_free_request(req
);
971 TAILQ_FOREACH_SAFE(qpair
, &proc
->allocated_io_qpairs
, per_process_tailq
, tmp_qpair
) {
972 TAILQ_REMOVE(&proc
->allocated_io_qpairs
, qpair
, per_process_tailq
);
975 * The process may have been killed while some qpairs were in their
976 * completion context. Clear that flag here to allow these IO
977 * qpairs to be deleted.
979 qpair
->in_completion_context
= 0;
980 spdk_nvme_ctrlr_free_io_qpair(qpair
);
987 * This function will be called when destructing the controller.
988 * 1. There is no more admin request on this controller.
989 * 2. Clean up any left resource allocation when its associated process is gone.
992 nvme_ctrlr_free_processes(struct spdk_nvme_ctrlr
*ctrlr
)
994 struct spdk_nvme_ctrlr_process
*active_proc
, *tmp
;
996 /* Free all the processes' properties and make sure no pending admin IOs */
997 TAILQ_FOREACH_SAFE(active_proc
, &ctrlr
->active_procs
, tailq
, tmp
) {
998 TAILQ_REMOVE(&ctrlr
->active_procs
, active_proc
, tailq
);
1000 assert(STAILQ_EMPTY(&active_proc
->active_reqs
));
1002 spdk_free(active_proc
);
1007 * This function will be called when any other process attaches or
1008 * detaches the controller in order to cleanup those unexpectedly
1009 * terminated processes.
1010 * Note: the ctrlr_lock must be held when calling this function.
1013 nvme_ctrlr_remove_inactive_proc(struct spdk_nvme_ctrlr
*ctrlr
)
1015 struct spdk_nvme_ctrlr_process
*active_proc
, *tmp
;
1016 int active_proc_count
= 0;
1018 TAILQ_FOREACH_SAFE(active_proc
, &ctrlr
->active_procs
, tailq
, tmp
) {
1019 if ((kill(active_proc
->pid
, 0) == -1) && (errno
== ESRCH
)) {
1020 SPDK_ERRLOG("process %d terminated unexpected\n", active_proc
->pid
);
1022 TAILQ_REMOVE(&ctrlr
->active_procs
, active_proc
, tailq
);
1024 nvme_ctrlr_cleanup_process(active_proc
);
1026 active_proc_count
++;
1030 return active_proc_count
;
1034 nvme_ctrlr_proc_get_ref(struct spdk_nvme_ctrlr
*ctrlr
)
1036 struct spdk_nvme_ctrlr_process
*active_proc
;
1037 pid_t pid
= getpid();
1039 nvme_robust_mutex_lock(&ctrlr
->ctrlr_lock
);
1041 nvme_ctrlr_remove_inactive_proc(ctrlr
);
1043 TAILQ_FOREACH(active_proc
, &ctrlr
->active_procs
, tailq
) {
1044 if (active_proc
->pid
== pid
) {
1050 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
1054 nvme_ctrlr_proc_put_ref(struct spdk_nvme_ctrlr
*ctrlr
)
1056 struct spdk_nvme_ctrlr_process
*active_proc
, *tmp
;
1057 pid_t pid
= getpid();
1060 nvme_robust_mutex_lock(&ctrlr
->ctrlr_lock
);
1062 proc_count
= nvme_ctrlr_remove_inactive_proc(ctrlr
);
1064 TAILQ_FOREACH_SAFE(active_proc
, &ctrlr
->active_procs
, tailq
, tmp
) {
1065 if (active_proc
->pid
== pid
) {
1067 assert(active_proc
->ref
>= 0);
1070 * The last active process will be removed at the end of
1071 * the destruction of the controller.
1073 if (active_proc
->ref
== 0 && proc_count
!= 1) {
1074 nvme_ctrlr_remove_process(ctrlr
, active_proc
);
1081 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
1085 nvme_ctrlr_get_ref_count(struct spdk_nvme_ctrlr
*ctrlr
)
1087 struct spdk_nvme_ctrlr_process
*active_proc
;
1090 nvme_robust_mutex_lock(&ctrlr
->ctrlr_lock
);
1092 nvme_ctrlr_remove_inactive_proc(ctrlr
);
1094 TAILQ_FOREACH(active_proc
, &ctrlr
->active_procs
, tailq
) {
1095 ref
+= active_proc
->ref
;
1098 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
1104 * This function will be called repeatedly during initialization until the controller is ready.
1107 nvme_ctrlr_process_init(struct spdk_nvme_ctrlr
*ctrlr
)
1109 union spdk_nvme_cc_register cc
;
1110 union spdk_nvme_csts_register csts
;
1111 uint32_t ready_timeout_in_ms
;
1115 * May need to avoid accessing any register on the target controller
1116 * for a while. Return early without touching the FSM.
1117 * Check sleep_timeout_tsc > 0 for unit test.
1119 if ((ctrlr
->sleep_timeout_tsc
> 0) &&
1120 (spdk_get_ticks() <= ctrlr
->sleep_timeout_tsc
)) {
1123 ctrlr
->sleep_timeout_tsc
= 0;
1125 if (nvme_ctrlr_get_cc(ctrlr
, &cc
) ||
1126 nvme_ctrlr_get_csts(ctrlr
, &csts
)) {
1127 if (ctrlr
->state_timeout_tsc
!= NVME_TIMEOUT_INFINITE
) {
1128 /* While a device is resetting, it may be unable to service MMIO reads
1129 * temporarily. Allow for this case.
1131 SPDK_TRACELOG(SPDK_TRACE_NVME
, "Get registers failed while waiting for CSTS.RDY == 0\n");
1134 SPDK_ERRLOG("Failed to read CC and CSTS in state %d\n", ctrlr
->state
);
1135 nvme_ctrlr_fail(ctrlr
, false);
1139 ready_timeout_in_ms
= 500 * ctrlr
->cap
.bits
.to
;
1142 * Check if the current initialization step is done or has timed out.
1144 switch (ctrlr
->state
) {
1145 case NVME_CTRLR_STATE_INIT
:
1146 /* Begin the hardware initialization by making sure the controller is disabled. */
1148 SPDK_TRACELOG(SPDK_TRACE_NVME
, "CC.EN = 1\n");
1150 * Controller is currently enabled. We need to disable it to cause a reset.
1152 * If CC.EN = 1 && CSTS.RDY = 0, the controller is in the process of becoming ready.
1153 * Wait for the ready bit to be 1 before disabling the controller.
1155 if (csts
.bits
.rdy
== 0) {
1156 SPDK_TRACELOG(SPDK_TRACE_NVME
, "CC.EN = 1 && CSTS.RDY = 0 - waiting for reset to complete\n");
1157 nvme_ctrlr_set_state(ctrlr
, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1
, ready_timeout_in_ms
);
1161 /* CC.EN = 1 && CSTS.RDY == 1, so we can immediately disable the controller. */
1162 SPDK_TRACELOG(SPDK_TRACE_NVME
, "Setting CC.EN = 0\n");
1164 if (nvme_ctrlr_set_cc(ctrlr
, &cc
)) {
1165 SPDK_ERRLOG("set_cc() failed\n");
1166 nvme_ctrlr_fail(ctrlr
, false);
1169 nvme_ctrlr_set_state(ctrlr
, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0
, ready_timeout_in_ms
);
1172 * Wait 2 secsonds before accessing PCI registers.
1173 * Not using sleep() to avoid blocking other controller's initialization.
1175 if (ctrlr
->quirks
& NVME_QUIRK_DELAY_BEFORE_CHK_RDY
) {
1176 SPDK_TRACELOG(SPDK_TRACE_NVME
, "Applying quirk: delay 2 seconds before reading registers\n");
1177 ctrlr
->sleep_timeout_tsc
= spdk_get_ticks() + 2 * spdk_get_ticks_hz();
1181 if (csts
.bits
.rdy
== 1) {
1182 SPDK_TRACELOG(SPDK_TRACE_NVME
, "CC.EN = 0 && CSTS.RDY = 1 - waiting for shutdown to complete\n");
1185 nvme_ctrlr_set_state(ctrlr
, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0
, ready_timeout_in_ms
);
1190 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1
:
1191 if (csts
.bits
.rdy
== 1) {
1192 SPDK_TRACELOG(SPDK_TRACE_NVME
, "CC.EN = 1 && CSTS.RDY = 1 - disabling controller\n");
1193 /* CC.EN = 1 && CSTS.RDY = 1, so we can set CC.EN = 0 now. */
1194 SPDK_TRACELOG(SPDK_TRACE_NVME
, "Setting CC.EN = 0\n");
1196 if (nvme_ctrlr_set_cc(ctrlr
, &cc
)) {
1197 SPDK_ERRLOG("set_cc() failed\n");
1198 nvme_ctrlr_fail(ctrlr
, false);
1201 nvme_ctrlr_set_state(ctrlr
, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0
, ready_timeout_in_ms
);
1206 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0
:
1207 if (csts
.bits
.rdy
== 0) {
1208 SPDK_TRACELOG(SPDK_TRACE_NVME
, "CC.EN = 0 && CSTS.RDY = 0\n");
1209 nvme_ctrlr_set_state(ctrlr
, NVME_CTRLR_STATE_ENABLE
, ready_timeout_in_ms
);
1211 if (ctrlr
->quirks
& NVME_QUIRK_DELAY_BEFORE_ENABLE
) {
1212 SPDK_TRACELOG(SPDK_TRACE_NVME
, "Applying quirk: Delay 100us before enabling.\n");
1213 ctrlr
->sleep_timeout_tsc
= spdk_get_ticks() + spdk_get_ticks_hz() / 10000;
1220 case NVME_CTRLR_STATE_ENABLE
:
1221 SPDK_TRACELOG(SPDK_TRACE_NVME
, "Setting CC.EN = 1\n");
1222 rc
= nvme_ctrlr_enable(ctrlr
);
1223 nvme_ctrlr_set_state(ctrlr
, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1
, ready_timeout_in_ms
);
1226 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1
:
1227 if (csts
.bits
.rdy
== 1) {
1228 SPDK_TRACELOG(SPDK_TRACE_NVME
, "CC.EN = 1 && CSTS.RDY = 1 - controller is ready\n");
1230 * The controller has been enabled.
1231 * Perform the rest of initialization in nvme_ctrlr_start() serially.
1233 rc
= nvme_ctrlr_start(ctrlr
);
1234 nvme_ctrlr_set_state(ctrlr
, NVME_CTRLR_STATE_READY
, NVME_TIMEOUT_INFINITE
);
1241 nvme_ctrlr_fail(ctrlr
, false);
1246 if (ctrlr
->state_timeout_tsc
!= NVME_TIMEOUT_INFINITE
&&
1247 spdk_get_ticks() > ctrlr
->state_timeout_tsc
) {
1248 SPDK_ERRLOG("Initialization timed out in state %d\n", ctrlr
->state
);
1249 nvme_ctrlr_fail(ctrlr
, false);
1257 nvme_ctrlr_start(struct spdk_nvme_ctrlr
*ctrlr
)
1259 nvme_transport_qpair_reset(ctrlr
->adminq
);
1261 nvme_qpair_enable(ctrlr
->adminq
);
1263 if (nvme_ctrlr_identify(ctrlr
) != 0) {
1267 if (nvme_ctrlr_set_num_qpairs(ctrlr
) != 0) {
1271 if (nvme_ctrlr_construct_namespaces(ctrlr
) != 0) {
1275 if (nvme_ctrlr_configure_aer(ctrlr
) != 0) {
1279 nvme_ctrlr_set_supported_log_pages(ctrlr
);
1280 nvme_ctrlr_set_supported_features(ctrlr
);
1282 if (ctrlr
->cdata
.sgls
.supported
) {
1283 ctrlr
->flags
|= SPDK_NVME_CTRLR_SGL_SUPPORTED
;
1286 if (nvme_ctrlr_set_keep_alive_timeout(ctrlr
) != 0) {
1287 SPDK_ERRLOG("Setting keep alive timeout failed\n");
1295 nvme_robust_mutex_init_recursive_shared(pthread_mutex_t
*mtx
)
1297 pthread_mutexattr_t attr
;
1300 if (pthread_mutexattr_init(&attr
)) {
1303 if (pthread_mutexattr_settype(&attr
, PTHREAD_MUTEX_RECURSIVE
) ||
1305 pthread_mutexattr_setrobust(&attr
, PTHREAD_MUTEX_ROBUST
) ||
1306 pthread_mutexattr_setpshared(&attr
, PTHREAD_PROCESS_SHARED
) ||
1308 pthread_mutex_init(mtx
, &attr
)) {
1311 pthread_mutexattr_destroy(&attr
);
1316 nvme_ctrlr_construct(struct spdk_nvme_ctrlr
*ctrlr
)
1320 nvme_ctrlr_set_state(ctrlr
, NVME_CTRLR_STATE_INIT
, NVME_TIMEOUT_INFINITE
);
1322 ctrlr
->free_io_qids
= NULL
;
1323 ctrlr
->is_resetting
= false;
1324 ctrlr
->is_failed
= false;
1326 TAILQ_INIT(&ctrlr
->active_io_qpairs
);
1327 STAILQ_INIT(&ctrlr
->queued_aborts
);
1328 ctrlr
->outstanding_aborts
= 0;
1330 rc
= nvme_robust_mutex_init_recursive_shared(&ctrlr
->ctrlr_lock
);
1335 TAILQ_INIT(&ctrlr
->active_procs
);
1336 ctrlr
->timeout_cb_fn
= NULL
;
1337 ctrlr
->timeout_cb_arg
= NULL
;
1338 ctrlr
->timeout_ticks
= 0;
1343 /* This function should be called once at ctrlr initialization to set up constant properties. */
1345 nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr
*ctrlr
, const union spdk_nvme_cap_register
*cap
)
1347 uint32_t max_io_queue_size
= nvme_transport_ctrlr_get_max_io_queue_size(ctrlr
);
1351 ctrlr
->min_page_size
= 1u << (12 + ctrlr
->cap
.bits
.mpsmin
);
1353 ctrlr
->opts
.io_queue_size
= spdk_max(ctrlr
->opts
.io_queue_size
, SPDK_NVME_IO_QUEUE_MIN_ENTRIES
);
1354 ctrlr
->opts
.io_queue_size
= spdk_min(ctrlr
->opts
.io_queue_size
, ctrlr
->cap
.bits
.mqes
+ 1u);
1355 ctrlr
->opts
.io_queue_size
= spdk_min(ctrlr
->opts
.io_queue_size
, max_io_queue_size
);
1357 ctrlr
->opts
.io_queue_requests
= spdk_max(ctrlr
->opts
.io_queue_requests
, ctrlr
->opts
.io_queue_size
);
1361 nvme_ctrlr_destruct(struct spdk_nvme_ctrlr
*ctrlr
)
1363 while (!TAILQ_EMPTY(&ctrlr
->active_io_qpairs
)) {
1364 struct spdk_nvme_qpair
*qpair
= TAILQ_FIRST(&ctrlr
->active_io_qpairs
);
1366 spdk_nvme_ctrlr_free_io_qpair(qpair
);
1369 nvme_ctrlr_shutdown(ctrlr
);
1371 nvme_ctrlr_destruct_namespaces(ctrlr
);
1373 spdk_bit_array_free(&ctrlr
->free_io_qids
);
1375 pthread_mutex_destroy(&ctrlr
->ctrlr_lock
);
1377 nvme_transport_ctrlr_destruct(ctrlr
);
1381 nvme_ctrlr_submit_admin_request(struct spdk_nvme_ctrlr
*ctrlr
,
1382 struct nvme_request
*req
)
1384 return nvme_qpair_submit_request(ctrlr
->adminq
, req
);
1388 nvme_keep_alive_completion(void *cb_ctx
, const struct spdk_nvme_cpl
*cpl
)
1394 * Check if we need to send a Keep Alive command.
1395 * Caller must hold ctrlr->ctrlr_lock.
1398 nvme_ctrlr_keep_alive(struct spdk_nvme_ctrlr
*ctrlr
)
1401 struct nvme_request
*req
;
1402 struct spdk_nvme_cmd
*cmd
;
1405 now
= spdk_get_ticks();
1406 if (now
< ctrlr
->next_keep_alive_tick
) {
1410 req
= nvme_allocate_request_null(ctrlr
->adminq
, nvme_keep_alive_completion
, NULL
);
1416 cmd
->opc
= SPDK_NVME_OPC_KEEP_ALIVE
;
1418 rc
= nvme_ctrlr_submit_admin_request(ctrlr
, req
);
1420 SPDK_ERRLOG("Submitting Keep Alive failed\n");
1423 ctrlr
->next_keep_alive_tick
= now
+ ctrlr
->keep_alive_interval_ticks
;
1427 spdk_nvme_ctrlr_process_admin_completions(struct spdk_nvme_ctrlr
*ctrlr
)
1429 int32_t num_completions
;
1431 nvme_robust_mutex_lock(&ctrlr
->ctrlr_lock
);
1432 if (ctrlr
->keep_alive_interval_ticks
) {
1433 nvme_ctrlr_keep_alive(ctrlr
);
1435 num_completions
= spdk_nvme_qpair_process_completions(ctrlr
->adminq
, 0);
1436 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
1438 return num_completions
;
1441 const struct spdk_nvme_ctrlr_data
*
1442 spdk_nvme_ctrlr_get_data(struct spdk_nvme_ctrlr
*ctrlr
)
1444 return &ctrlr
->cdata
;
1447 union spdk_nvme_csts_register
spdk_nvme_ctrlr_get_regs_csts(struct spdk_nvme_ctrlr
*ctrlr
)
1449 union spdk_nvme_csts_register csts
;
1451 if (nvme_ctrlr_get_csts(ctrlr
, &csts
)) {
1457 union spdk_nvme_cap_register
spdk_nvme_ctrlr_get_regs_cap(struct spdk_nvme_ctrlr
*ctrlr
)
1462 union spdk_nvme_vs_register
spdk_nvme_ctrlr_get_regs_vs(struct spdk_nvme_ctrlr
*ctrlr
)
1464 union spdk_nvme_vs_register vs
;
1466 if (nvme_ctrlr_get_vs(ctrlr
, &vs
)) {
1473 spdk_nvme_ctrlr_get_num_ns(struct spdk_nvme_ctrlr
*ctrlr
)
1475 return ctrlr
->num_ns
;
1478 struct spdk_nvme_ns
*
1479 spdk_nvme_ctrlr_get_ns(struct spdk_nvme_ctrlr
*ctrlr
, uint32_t ns_id
)
1481 if (ns_id
< 1 || ns_id
> ctrlr
->num_ns
) {
1485 return &ctrlr
->ns
[ns_id
- 1];
1489 spdk_nvme_ctrlr_register_aer_callback(struct spdk_nvme_ctrlr
*ctrlr
,
1490 spdk_nvme_aer_cb aer_cb_fn
,
1493 ctrlr
->aer_cb_fn
= aer_cb_fn
;
1494 ctrlr
->aer_cb_arg
= aer_cb_arg
;
1498 spdk_nvme_ctrlr_register_timeout_callback(struct spdk_nvme_ctrlr
*ctrlr
,
1499 uint32_t nvme_timeout
, spdk_nvme_timeout_cb cb_fn
, void *cb_arg
)
1501 ctrlr
->timeout_ticks
= nvme_timeout
* spdk_get_ticks_hz();
1502 ctrlr
->timeout_cb_fn
= cb_fn
;
1503 ctrlr
->timeout_cb_arg
= cb_arg
;
1507 spdk_nvme_ctrlr_is_log_page_supported(struct spdk_nvme_ctrlr
*ctrlr
, uint8_t log_page
)
1509 /* No bounds check necessary, since log_page is uint8_t and log_page_supported has 256 entries */
1510 SPDK_STATIC_ASSERT(sizeof(ctrlr
->log_page_supported
) == 256, "log_page_supported size mismatch");
1511 return ctrlr
->log_page_supported
[log_page
];
1515 spdk_nvme_ctrlr_is_feature_supported(struct spdk_nvme_ctrlr
*ctrlr
, uint8_t feature_code
)
1517 /* No bounds check necessary, since feature_code is uint8_t and feature_supported has 256 entries */
1518 SPDK_STATIC_ASSERT(sizeof(ctrlr
->feature_supported
) == 256, "feature_supported size mismatch");
1519 return ctrlr
->feature_supported
[feature_code
];
1523 spdk_nvme_ctrlr_attach_ns(struct spdk_nvme_ctrlr
*ctrlr
, uint32_t nsid
,
1524 struct spdk_nvme_ctrlr_list
*payload
)
1526 struct nvme_completion_poll_status status
;
1529 status
.done
= false;
1530 res
= nvme_ctrlr_cmd_attach_ns(ctrlr
, nsid
, payload
,
1531 nvme_completion_poll_cb
, &status
);
1534 while (status
.done
== false) {
1535 nvme_robust_mutex_lock(&ctrlr
->ctrlr_lock
);
1536 spdk_nvme_qpair_process_completions(ctrlr
->adminq
, 0);
1537 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
1539 if (spdk_nvme_cpl_is_error(&status
.cpl
)) {
1540 SPDK_ERRLOG("spdk_nvme_ctrlr_attach_ns failed!\n");
1544 return spdk_nvme_ctrlr_reset(ctrlr
);
1548 spdk_nvme_ctrlr_detach_ns(struct spdk_nvme_ctrlr
*ctrlr
, uint32_t nsid
,
1549 struct spdk_nvme_ctrlr_list
*payload
)
1551 struct nvme_completion_poll_status status
;
1554 status
.done
= false;
1555 res
= nvme_ctrlr_cmd_detach_ns(ctrlr
, nsid
, payload
,
1556 nvme_completion_poll_cb
, &status
);
1559 while (status
.done
== false) {
1560 nvme_robust_mutex_lock(&ctrlr
->ctrlr_lock
);
1561 spdk_nvme_qpair_process_completions(ctrlr
->adminq
, 0);
1562 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
1564 if (spdk_nvme_cpl_is_error(&status
.cpl
)) {
1565 SPDK_ERRLOG("spdk_nvme_ctrlr_detach_ns failed!\n");
1569 return spdk_nvme_ctrlr_reset(ctrlr
);
1573 spdk_nvme_ctrlr_create_ns(struct spdk_nvme_ctrlr
*ctrlr
, struct spdk_nvme_ns_data
*payload
)
1575 struct nvme_completion_poll_status status
;
1578 status
.done
= false;
1579 res
= nvme_ctrlr_cmd_create_ns(ctrlr
, payload
, nvme_completion_poll_cb
, &status
);
1582 while (status
.done
== false) {
1583 nvme_robust_mutex_lock(&ctrlr
->ctrlr_lock
);
1584 spdk_nvme_qpair_process_completions(ctrlr
->adminq
, 0);
1585 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
1587 if (spdk_nvme_cpl_is_error(&status
.cpl
)) {
1588 SPDK_ERRLOG("spdk_nvme_ctrlr_create_ns failed!\n");
1592 res
= spdk_nvme_ctrlr_reset(ctrlr
);
1597 /* Return the namespace ID that was created */
1598 return status
.cpl
.cdw0
;
1602 spdk_nvme_ctrlr_delete_ns(struct spdk_nvme_ctrlr
*ctrlr
, uint32_t nsid
)
1604 struct nvme_completion_poll_status status
;
1607 status
.done
= false;
1608 res
= nvme_ctrlr_cmd_delete_ns(ctrlr
, nsid
, nvme_completion_poll_cb
, &status
);
1611 while (status
.done
== false) {
1612 nvme_robust_mutex_lock(&ctrlr
->ctrlr_lock
);
1613 spdk_nvme_qpair_process_completions(ctrlr
->adminq
, 0);
1614 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
1616 if (spdk_nvme_cpl_is_error(&status
.cpl
)) {
1617 SPDK_ERRLOG("spdk_nvme_ctrlr_delete_ns failed!\n");
1621 return spdk_nvme_ctrlr_reset(ctrlr
);
1625 spdk_nvme_ctrlr_format(struct spdk_nvme_ctrlr
*ctrlr
, uint32_t nsid
,
1626 struct spdk_nvme_format
*format
)
1628 struct nvme_completion_poll_status status
;
1631 status
.done
= false;
1632 res
= nvme_ctrlr_cmd_format(ctrlr
, nsid
, format
, nvme_completion_poll_cb
,
1636 while (status
.done
== false) {
1637 nvme_robust_mutex_lock(&ctrlr
->ctrlr_lock
);
1638 spdk_nvme_qpair_process_completions(ctrlr
->adminq
, 0);
1639 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
1641 if (spdk_nvme_cpl_is_error(&status
.cpl
)) {
1642 SPDK_ERRLOG("spdk_nvme_ctrlr_format failed!\n");
1646 return spdk_nvme_ctrlr_reset(ctrlr
);
1650 spdk_nvme_ctrlr_update_firmware(struct spdk_nvme_ctrlr
*ctrlr
, void *payload
, uint32_t size
,
1653 struct spdk_nvme_fw_commit fw_commit
;
1654 struct nvme_completion_poll_status status
;
1656 unsigned int size_remaining
;
1657 unsigned int offset
;
1658 unsigned int transfer
;
1662 SPDK_ERRLOG("spdk_nvme_ctrlr_update_firmware invalid size!\n");
1666 /* Firmware download */
1667 size_remaining
= size
;
1671 while (size_remaining
> 0) {
1672 transfer
= spdk_min(size_remaining
, ctrlr
->min_page_size
);
1673 status
.done
= false;
1675 res
= nvme_ctrlr_cmd_fw_image_download(ctrlr
, transfer
, offset
, p
,
1676 nvme_completion_poll_cb
,
1681 while (status
.done
== false) {
1682 nvme_robust_mutex_lock(&ctrlr
->ctrlr_lock
);
1683 spdk_nvme_qpair_process_completions(ctrlr
->adminq
, 0);
1684 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
1686 if (spdk_nvme_cpl_is_error(&status
.cpl
)) {
1687 SPDK_ERRLOG("spdk_nvme_ctrlr_fw_image_download failed!\n");
1692 size_remaining
-= transfer
;
1695 /* Firmware commit */
1696 memset(&fw_commit
, 0, sizeof(struct spdk_nvme_fw_commit
));
1697 fw_commit
.fs
= slot
;
1698 fw_commit
.ca
= SPDK_NVME_FW_COMMIT_REPLACE_IMG
;
1700 status
.done
= false;
1702 res
= nvme_ctrlr_cmd_fw_commit(ctrlr
, &fw_commit
, nvme_completion_poll_cb
,
1707 while (status
.done
== false) {
1708 nvme_robust_mutex_lock(&ctrlr
->ctrlr_lock
);
1709 spdk_nvme_qpair_process_completions(ctrlr
->adminq
, 0);
1710 nvme_robust_mutex_unlock(&ctrlr
->ctrlr_lock
);
1712 if (spdk_nvme_cpl_is_error(&status
.cpl
)) {
1713 SPDK_ERRLOG("nvme_ctrlr_cmd_fw_commit failed!\n");
1717 return spdk_nvme_ctrlr_reset(ctrlr
);