4 * Copyright (c) Intel Corporation.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #ifndef __NVME_INTERNAL_H__
35 #define __NVME_INTERNAL_H__
37 #include "spdk/config.h"
38 #include "spdk/likely.h"
39 #include "spdk/stdinc.h"
41 #include "spdk/nvme.h"
43 #if defined(__i386__) || defined(__x86_64__)
44 #include <x86intrin.h>
47 #include "spdk/queue.h"
48 #include "spdk/barrier.h"
49 #include "spdk/bit_array.h"
50 #include "spdk/mmio.h"
51 #include "spdk/pci_ids.h"
52 #include "spdk/util.h"
53 #include "spdk/nvme_intel.h"
54 #include "spdk/nvmf_spec.h"
55 #include "spdk/uuid.h"
57 #include "spdk_internal/assert.h"
58 #include "spdk_internal/log.h"
59 #include "spdk_internal/memory.h"
61 extern pid_t g_spdk_nvme_pid
;
64 * Some Intel devices support vendor-unique read latency log page even
65 * though the log page directory says otherwise.
67 #define NVME_INTEL_QUIRK_READ_LATENCY 0x1
70 * Some Intel devices support vendor-unique write latency log page even
71 * though the log page directory says otherwise.
73 #define NVME_INTEL_QUIRK_WRITE_LATENCY 0x2
76 * The controller needs a delay before starts checking the device
77 * readiness, which is done by reading the NVME_CSTS_RDY bit.
79 #define NVME_QUIRK_DELAY_BEFORE_CHK_RDY 0x4
82 * The controller performs best when I/O is split on particular
85 #define NVME_INTEL_QUIRK_STRIPING 0x8
88 * The controller needs a delay after allocating an I/O queue pair
89 * before it is ready to accept I/O commands.
91 #define NVME_QUIRK_DELAY_AFTER_QUEUE_ALLOC 0x10
94 * Earlier NVMe devices do not indicate whether unmapped blocks
95 * will read all zeroes or not. This define indicates that the
96 * device does in fact read all zeroes after an unmap event
98 #define NVME_QUIRK_READ_ZERO_AFTER_DEALLOCATE 0x20
101 * The controller doesn't handle Identify value others than 0 or 1 correctly.
103 #define NVME_QUIRK_IDENTIFY_CNS 0x40
106 * The controller supports Open Channel command set if matching additional
107 * condition, like the first byte (value 0x1) in the vendor specific
108 * bits of the namespace identify structure is set.
110 #define NVME_QUIRK_OCSSD 0x80
113 * The controller has an Intel vendor ID but does not support Intel vendor-specific
114 * log pages. This is primarily for QEMU emulated SSDs which report an Intel vendor
115 * ID but do not support these log pages.
117 #define NVME_INTEL_QUIRK_NO_LOG_PAGES 0x100
120 * The controller does not set SHST_COMPLETE in a reasonable amount of time. This
121 * is primarily seen in virtual VMWare NVMe SSDs. This quirk merely adds an additional
122 * error message that on VMWare NVMe SSDs, the shutdown timeout may be expected.
124 #define NVME_QUIRK_SHST_COMPLETE 0x200
126 #define NVME_MAX_ASYNC_EVENTS (8)
128 #define NVME_MAX_ADMIN_TIMEOUT_IN_SECS (30)
130 /* Maximum log page size to fetch for AERs. */
131 #define NVME_MAX_AER_LOG_SIZE (4096)
134 * NVME_MAX_IO_QUEUES in nvme_spec.h defines the 64K spec-limit, but this
135 * define specifies the maximum number of queues this driver will actually
136 * try to configure, if available.
138 #define DEFAULT_MAX_IO_QUEUES (1024)
139 #define DEFAULT_IO_QUEUE_SIZE (256)
141 #define DEFAULT_ADMIN_QUEUE_REQUESTS (32)
142 #define DEFAULT_IO_QUEUE_REQUESTS (512)
144 #define MIN_KEEP_ALIVE_TIMEOUT_IN_MS (10000)
146 /* We want to fit submission and completion rings each in a single 2MB
147 * hugepage to ensure physical address contiguity.
149 #define MAX_IO_QUEUE_ENTRIES (VALUE_2MB / spdk_max( \
150 sizeof(struct spdk_nvme_cmd), \
151 sizeof(struct spdk_nvme_cpl)))
153 enum nvme_payload_type
{
154 NVME_PAYLOAD_TYPE_INVALID
= 0,
156 /** nvme_request::u.payload.contig_buffer is valid for this request */
157 NVME_PAYLOAD_TYPE_CONTIG
,
159 /** nvme_request::u.sgl is valid for this request */
160 NVME_PAYLOAD_TYPE_SGL
,
164 * Descriptor for a request data payload.
166 struct nvme_payload
{
168 * Functions for retrieving physical addresses for scattered payloads.
170 spdk_nvme_req_reset_sgl_cb reset_sgl_fn
;
171 spdk_nvme_req_next_sge_cb next_sge_fn
;
174 * If reset_sgl_fn == NULL, this is a contig payload, and contig_or_cb_arg contains the
175 * virtual memory address of a single virtually contiguous buffer.
177 * If reset_sgl_fn != NULL, this is a SGL payload, and contig_or_cb_arg contains the
178 * cb_arg that will be passed to the SGL callback functions.
180 void *contig_or_cb_arg
;
182 /** Virtual memory address of a single virtually contiguous metadata buffer */
186 #define NVME_PAYLOAD_CONTIG(contig_, md_) \
187 (struct nvme_payload) { \
188 .reset_sgl_fn = NULL, \
189 .next_sge_fn = NULL, \
190 .contig_or_cb_arg = (contig_), \
194 #define NVME_PAYLOAD_SGL(reset_sgl_fn_, next_sge_fn_, cb_arg_, md_) \
195 (struct nvme_payload) { \
196 .reset_sgl_fn = (reset_sgl_fn_), \
197 .next_sge_fn = (next_sge_fn_), \
198 .contig_or_cb_arg = (cb_arg_), \
202 static inline enum nvme_payload_type
203 nvme_payload_type(const struct nvme_payload
*payload
) {
204 return payload
->reset_sgl_fn
? NVME_PAYLOAD_TYPE_SGL
: NVME_PAYLOAD_TYPE_CONTIG
;
207 struct nvme_error_cmd
{
209 uint64_t timeout_tsc
;
212 struct spdk_nvme_status status
;
213 TAILQ_ENTRY(nvme_error_cmd
) link
;
216 struct nvme_request
{
217 struct spdk_nvme_cmd cmd
;
224 * Number of children requests still outstanding for this
225 * request which was split into multiple child requests.
227 uint16_t num_children
;
230 * Offset in bytes from the beginning of payload for this request.
231 * This is used for I/O commands that are split into multiple requests.
233 uint32_t payload_offset
;
236 uint32_t payload_size
;
239 * Timeout ticks for error injection requests, can be extended in future
240 * to support per-request timeout feature.
242 uint64_t timeout_tsc
;
245 * Data payload for this request's command.
247 struct nvme_payload payload
;
249 spdk_nvme_cmd_cb cb_fn
;
251 STAILQ_ENTRY(nvme_request
) stailq
;
253 struct spdk_nvme_qpair
*qpair
;
256 * The value of spdk_get_ticks() when the request was submitted to the hardware.
257 * Only set if ctrlr->timeout_enabled is true.
259 uint64_t submit_tick
;
262 * The active admin request can be moved to a per process pending
263 * list based on the saved pid to tell which process it belongs
264 * to. The cpl saves the original completion information which
265 * is used in the completion callback.
266 * NOTE: these below two fields are only used for admin request.
269 struct spdk_nvme_cpl cpl
;
272 * The following members should not be reordered with members
273 * above. These members are only needed when splitting
274 * requests which is done rarely, and the driver is careful
275 * to not touch the following fields until a split operation is
276 * needed, to avoid touching an extra cacheline.
280 * Points to the outstanding child requests for a parent request.
281 * Only valid if a request was split into multiple children
282 * requests, and is not initialized for non-split requests.
284 TAILQ_HEAD(, nvme_request
) children
;
287 * Linked-list pointers for a child request in its parent's list.
289 TAILQ_ENTRY(nvme_request
) child_tailq
;
292 * Points to a parent request if part of a split request,
295 struct nvme_request
*parent
;
298 * Completion status for a parent request. Initialized to all 0's
299 * (SUCCESS) before child requests are submitted. If a child
300 * request completes with error, the error status is copied here,
301 * to ensure that the parent request is also completed with error
302 * status once all child requests are completed.
304 struct spdk_nvme_cpl parent_status
;
307 * The user_cb_fn and user_cb_arg fields are used for holding the original
308 * callback data when using nvme_allocate_request_user_copy.
310 spdk_nvme_cmd_cb user_cb_fn
;
315 struct nvme_completion_poll_status
{
316 struct spdk_nvme_cpl cpl
;
320 struct nvme_async_event_request
{
321 struct spdk_nvme_ctrlr
*ctrlr
;
322 struct nvme_request
*req
;
323 struct spdk_nvme_cpl cpl
;
326 struct spdk_nvme_qpair
{
327 struct spdk_nvme_ctrlr
*ctrlr
;
333 uint8_t is_enabled
: 1;
334 uint8_t is_connecting
: 1;
337 * Members for handling IO qpair deletion inside of a completion context.
338 * These are specifically defined as single bits, so that they do not
339 * push this data structure out to another cacheline.
341 uint8_t in_completion_context
: 1;
342 uint8_t delete_after_completion_context
: 1;
345 * Set when no deletion notification is needed. For example, the process
346 * which allocated this qpair exited unexpectedly.
348 uint8_t no_deletion_notification_needed
: 1;
350 enum spdk_nvme_transport_type trtype
;
352 STAILQ_HEAD(, nvme_request
) free_req
;
353 STAILQ_HEAD(, nvme_request
) queued_req
;
355 /** Commands opcode in this list will return error */
356 TAILQ_HEAD(, nvme_error_cmd
) err_cmd_head
;
357 /** Requests in this list will return error */
358 STAILQ_HEAD(, nvme_request
) err_req_head
;
360 /* List entry for spdk_nvme_ctrlr::active_io_qpairs */
361 TAILQ_ENTRY(spdk_nvme_qpair
) tailq
;
363 /* List entry for spdk_nvme_ctrlr_process::allocated_io_qpairs */
364 TAILQ_ENTRY(spdk_nvme_qpair
) per_process_tailq
;
366 struct spdk_nvme_ctrlr_process
*active_proc
;
371 struct spdk_nvme_ns
{
372 struct spdk_nvme_ctrlr
*ctrlr
;
373 uint32_t sector_size
;
376 * Size of data transferred as part of each block,
377 * including metadata if FLBAS indicates the metadata is transferred
378 * as part of the data buffer at the end of each LBA.
380 uint32_t extended_lba_size
;
384 uint32_t sectors_per_max_io
;
385 uint32_t sectors_per_stripe
;
389 /* Namespace Identification Descriptor List (CNS = 03h) */
390 uint8_t id_desc_list
[4096];
394 * State of struct spdk_nvme_ctrlr (in particular, during initialization).
396 enum nvme_ctrlr_state
{
398 * Wait before initializing the controller.
400 NVME_CTRLR_STATE_INIT_DELAY
,
403 * Controller has not been initialized yet.
405 NVME_CTRLR_STATE_INIT
,
408 * Waiting for CSTS.RDY to transition from 0 to 1 so that CC.EN may be set to 0.
410 NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1
,
413 * Waiting for CSTS.RDY to transition from 1 to 0 so that CC.EN may be set to 1.
415 NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0
,
418 * Enable the controller by writing CC.EN to 1
420 NVME_CTRLR_STATE_ENABLE
,
423 * Waiting for CSTS.RDY to transition from 0 to 1 after enabling the controller.
425 NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1
,
428 * Enable the Admin queue of the controller.
430 NVME_CTRLR_STATE_ENABLE_ADMIN_QUEUE
,
433 * Identify Controller command will be sent to then controller.
435 NVME_CTRLR_STATE_IDENTIFY
,
438 * Waiting for Identify Controller command be completed.
440 NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY
,
443 * Set Number of Queues of the controller.
445 NVME_CTRLR_STATE_SET_NUM_QUEUES
,
448 * Waiting for Set Num of Queues command to be completed.
450 NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES
,
453 * Get Number of Queues of the controller.
455 NVME_CTRLR_STATE_GET_NUM_QUEUES
,
458 * Waiting for Get Num of Queues command to be completed.
460 NVME_CTRLR_STATE_WAIT_FOR_GET_NUM_QUEUES
,
463 * Construct Namespace data structures of the controller.
465 NVME_CTRLR_STATE_CONSTRUCT_NS
,
468 * Get active Namespace list of the controller.
470 NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS
,
473 * Get Identify Namespace Data structure for each NS.
475 NVME_CTRLR_STATE_IDENTIFY_NS
,
478 * Waiting for the Identify Namespace commands to be completed.
480 NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS
,
483 * Get Identify Namespace Identification Descriptors.
485 NVME_CTRLR_STATE_IDENTIFY_ID_DESCS
,
488 * Waiting for the Identify Namespace Identification
489 * Descriptors to be completed.
491 NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS
,
494 * Configure AER of the controller.
496 NVME_CTRLR_STATE_CONFIGURE_AER
,
499 * Waiting for the Configure AER to be completed.
501 NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER
,
504 * Set supported log pages of the controller.
506 NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES
,
509 * Set supported features of the controller.
511 NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES
,
514 * Set Doorbell Buffer Config of the controller.
516 NVME_CTRLR_STATE_SET_DB_BUF_CFG
,
519 * Waiting for Doorbell Buffer Config to be completed.
521 NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG
,
524 * Set Keep Alive Timeout of the controller.
526 NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT
,
529 * Waiting for Set Keep Alive Timeout to be completed.
531 NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT
,
534 * Set Host ID of the controller.
536 NVME_CTRLR_STATE_SET_HOST_ID
,
539 * Waiting for Set Host ID to be completed.
541 NVME_CTRLR_STATE_WAIT_FOR_HOST_ID
,
544 * Controller initialization has completed and the controller is ready.
546 NVME_CTRLR_STATE_READY
,
549 * Controller inilialization has an error.
551 NVME_CTRLR_STATE_ERROR
554 #define NVME_TIMEOUT_INFINITE UINT64_MAX
557 * Used to track properties for all processes accessing the controller.
559 struct spdk_nvme_ctrlr_process
{
560 /** Whether it is the primary process */
566 /** Active admin requests to be completed */
567 STAILQ_HEAD(, nvme_request
) active_reqs
;
569 TAILQ_ENTRY(spdk_nvme_ctrlr_process
) tailq
;
571 /** Per process PCI device handle */
572 struct spdk_pci_device
*devhandle
;
574 /** Reference to track the number of attachment to this controller. */
577 /** Allocated IO qpairs */
578 TAILQ_HEAD(, spdk_nvme_qpair
) allocated_io_qpairs
;
580 spdk_nvme_aer_cb aer_cb_fn
;
584 * A function pointer to timeout callback function
586 spdk_nvme_timeout_cb timeout_cb_fn
;
587 void *timeout_cb_arg
;
588 uint64_t timeout_ticks
;
592 * One of these per allocated PCI device.
594 struct spdk_nvme_ctrlr
{
595 /* Hot data (accessed in I/O path) starts here. */
597 /** Array of namespaces indexed by nsid - 1 */
598 struct spdk_nvme_ns
*ns
;
600 struct spdk_nvme_transport_id trid
;
612 bool timeout_enabled
;
618 /** Controller support flags */
621 /* Cold data (not accessed in normal I/O path) is after this point. */
623 union spdk_nvme_cap_register cap
;
624 union spdk_nvme_vs_register vs
;
626 enum nvme_ctrlr_state state
;
627 uint64_t state_timeout_tsc
;
629 uint64_t next_keep_alive_tick
;
630 uint64_t keep_alive_interval_ticks
;
632 TAILQ_ENTRY(spdk_nvme_ctrlr
) tailq
;
634 /** All the log pages supported */
635 bool log_page_supported
[256];
637 /** All the features supported */
638 bool feature_supported
[256];
640 /** maximum i/o size in bytes */
641 uint32_t max_xfer_size
;
643 /** minimum page size supported by this controller in bytes */
644 uint32_t min_page_size
;
646 /** selected memory page size for this controller in bytes */
650 struct nvme_async_event_request aer
[NVME_MAX_ASYNC_EVENTS
];
652 /** guards access to the controller itself, including admin queues */
653 pthread_mutex_t ctrlr_lock
;
656 struct spdk_nvme_qpair
*adminq
;
658 /** shadow doorbell buffer */
659 uint32_t *shadow_doorbell
;
660 /** eventidx buffer */
664 * Identify Controller data.
666 struct spdk_nvme_ctrlr_data cdata
;
669 * Keep track of active namespaces
671 uint32_t *active_ns_list
;
674 * Array of Identify Namespace data.
676 * Stored separately from ns since nsdata should not normally be accessed during I/O.
678 struct spdk_nvme_ns_data
*nsdata
;
680 struct spdk_bit_array
*free_io_qids
;
681 TAILQ_HEAD(, spdk_nvme_qpair
) active_io_qpairs
;
683 struct spdk_nvme_ctrlr_opts opts
;
687 /* Extra sleep time during controller initialization */
688 uint64_t sleep_timeout_tsc
;
690 /** Track all the processes manage this controller */
691 TAILQ_HEAD(, spdk_nvme_ctrlr_process
) active_procs
;
694 STAILQ_HEAD(, nvme_request
) queued_aborts
;
695 uint32_t outstanding_aborts
;
698 struct spdk_nvme_probe_ctx
{
699 struct spdk_nvme_transport_id trid
;
701 spdk_nvme_probe_cb probe_cb
;
702 spdk_nvme_attach_cb attach_cb
;
703 spdk_nvme_remove_cb remove_cb
;
704 TAILQ_HEAD(, spdk_nvme_ctrlr
) init_ctrlrs
;
708 pthread_mutex_t lock
;
710 /** Multi-process shared attached controller list */
711 TAILQ_HEAD(, spdk_nvme_ctrlr
) shared_attached_ctrlrs
;
714 struct spdk_uuid default_extended_host_id
;
717 extern struct nvme_driver
*g_spdk_nvme_driver
;
719 int nvme_driver_init(void);
721 #define nvme_delay usleep
724 nvme_qpair_is_admin_queue(struct spdk_nvme_qpair
*qpair
)
726 return qpair
->id
== 0;
730 nvme_qpair_is_io_queue(struct spdk_nvme_qpair
*qpair
)
732 return qpair
->id
!= 0;
736 nvme_robust_mutex_lock(pthread_mutex_t
*mtx
)
738 int rc
= pthread_mutex_lock(mtx
);
741 if (rc
== EOWNERDEAD
) {
742 rc
= pthread_mutex_consistent(mtx
);
750 nvme_robust_mutex_unlock(pthread_mutex_t
*mtx
)
752 return pthread_mutex_unlock(mtx
);
755 /* Admin functions */
756 int nvme_ctrlr_cmd_identify(struct spdk_nvme_ctrlr
*ctrlr
,
757 uint8_t cns
, uint16_t cntid
, uint32_t nsid
,
758 void *payload
, size_t payload_size
,
759 spdk_nvme_cmd_cb cb_fn
, void *cb_arg
);
760 int nvme_ctrlr_cmd_set_num_queues(struct spdk_nvme_ctrlr
*ctrlr
,
761 uint32_t num_queues
, spdk_nvme_cmd_cb cb_fn
,
763 int nvme_ctrlr_cmd_get_num_queues(struct spdk_nvme_ctrlr
*ctrlr
,
764 spdk_nvme_cmd_cb cb_fn
, void *cb_arg
);
765 int nvme_ctrlr_cmd_set_async_event_config(struct spdk_nvme_ctrlr
*ctrlr
,
766 union spdk_nvme_feat_async_event_configuration config
,
767 spdk_nvme_cmd_cb cb_fn
, void *cb_arg
);
768 int nvme_ctrlr_cmd_set_host_id(struct spdk_nvme_ctrlr
*ctrlr
, void *host_id
, uint32_t host_id_size
,
769 spdk_nvme_cmd_cb cb_fn
, void *cb_arg
);
770 int nvme_ctrlr_cmd_attach_ns(struct spdk_nvme_ctrlr
*ctrlr
, uint32_t nsid
,
771 struct spdk_nvme_ctrlr_list
*payload
, spdk_nvme_cmd_cb cb_fn
, void *cb_arg
);
772 int nvme_ctrlr_cmd_detach_ns(struct spdk_nvme_ctrlr
*ctrlr
, uint32_t nsid
,
773 struct spdk_nvme_ctrlr_list
*payload
, spdk_nvme_cmd_cb cb_fn
, void *cb_arg
);
774 int nvme_ctrlr_cmd_create_ns(struct spdk_nvme_ctrlr
*ctrlr
, struct spdk_nvme_ns_data
*payload
,
775 spdk_nvme_cmd_cb cb_fn
, void *cb_arg
);
776 int nvme_ctrlr_cmd_doorbell_buffer_config(struct spdk_nvme_ctrlr
*ctrlr
,
777 uint64_t prp1
, uint64_t prp2
,
778 spdk_nvme_cmd_cb cb_fn
, void *cb_arg
);
779 int nvme_ctrlr_cmd_delete_ns(struct spdk_nvme_ctrlr
*ctrlr
, uint32_t nsid
, spdk_nvme_cmd_cb cb_fn
,
781 int nvme_ctrlr_cmd_format(struct spdk_nvme_ctrlr
*ctrlr
, uint32_t nsid
,
782 struct spdk_nvme_format
*format
, spdk_nvme_cmd_cb cb_fn
, void *cb_arg
);
783 int nvme_ctrlr_cmd_fw_commit(struct spdk_nvme_ctrlr
*ctrlr
,
784 const struct spdk_nvme_fw_commit
*fw_commit
,
785 spdk_nvme_cmd_cb cb_fn
, void *cb_arg
);
786 int nvme_ctrlr_cmd_fw_image_download(struct spdk_nvme_ctrlr
*ctrlr
,
787 uint32_t size
, uint32_t offset
, void *payload
,
788 spdk_nvme_cmd_cb cb_fn
, void *cb_arg
);
789 int nvme_ctrlr_cmd_security_receive(struct spdk_nvme_ctrlr
*ctrlr
, uint8_t secp
, uint16_t spsp
,
790 uint8_t nssf
, void *payload
, uint32_t payload_size
,
791 spdk_nvme_cmd_cb cb_fn
, void *cb_arg
);
792 int nvme_ctrlr_cmd_security_send(struct spdk_nvme_ctrlr
*ctrlr
, uint8_t secp
,
793 uint16_t spsp
, uint8_t nssf
, void *payload
,
794 uint32_t payload_size
, spdk_nvme_cmd_cb cb_fn
, void *cb_arg
);
795 int nvme_ctrlr_cmd_sanitize(struct spdk_nvme_ctrlr
*ctrlr
, uint32_t nsid
,
796 struct spdk_nvme_sanitize
*sanitize
, uint32_t cdw11
,
797 spdk_nvme_cmd_cb cb_fn
, void *cb_arg
);
798 void nvme_completion_poll_cb(void *arg
, const struct spdk_nvme_cpl
*cpl
);
799 int spdk_nvme_wait_for_completion(struct spdk_nvme_qpair
*qpair
,
800 struct nvme_completion_poll_status
*status
);
801 int spdk_nvme_wait_for_completion_robust_lock(struct spdk_nvme_qpair
*qpair
,
802 struct nvme_completion_poll_status
*status
,
803 pthread_mutex_t
*robust_mutex
);
804 int spdk_nvme_wait_for_completion_timeout(struct spdk_nvme_qpair
*qpair
,
805 struct nvme_completion_poll_status
*status
,
806 uint64_t timeout_in_secs
);
808 struct spdk_nvme_ctrlr_process
*spdk_nvme_ctrlr_get_process(struct spdk_nvme_ctrlr
*ctrlr
,
810 struct spdk_nvme_ctrlr_process
*spdk_nvme_ctrlr_get_current_process(struct spdk_nvme_ctrlr
*ctrlr
);
811 int nvme_ctrlr_add_process(struct spdk_nvme_ctrlr
*ctrlr
, void *devhandle
);
812 void nvme_ctrlr_free_processes(struct spdk_nvme_ctrlr
*ctrlr
);
813 struct spdk_pci_device
*nvme_ctrlr_proc_get_devhandle(struct spdk_nvme_ctrlr
*ctrlr
);
815 int nvme_ctrlr_probe(const struct spdk_nvme_transport_id
*trid
,
816 struct spdk_nvme_probe_ctx
*probe_ctx
, void *devhandle
);
818 int nvme_ctrlr_construct(struct spdk_nvme_ctrlr
*ctrlr
);
819 void nvme_ctrlr_destruct_finish(struct spdk_nvme_ctrlr
*ctrlr
);
820 void nvme_ctrlr_destruct(struct spdk_nvme_ctrlr
*ctrlr
);
821 void nvme_ctrlr_fail(struct spdk_nvme_ctrlr
*ctrlr
, bool hot_remove
);
822 int nvme_ctrlr_process_init(struct spdk_nvme_ctrlr
*ctrlr
);
823 void nvme_ctrlr_connected(struct spdk_nvme_probe_ctx
*probe_ctx
,
824 struct spdk_nvme_ctrlr
*ctrlr
);
826 int nvme_ctrlr_submit_admin_request(struct spdk_nvme_ctrlr
*ctrlr
,
827 struct nvme_request
*req
);
828 int nvme_ctrlr_get_cap(struct spdk_nvme_ctrlr
*ctrlr
, union spdk_nvme_cap_register
*cap
);
829 int nvme_ctrlr_get_vs(struct spdk_nvme_ctrlr
*ctrlr
, union spdk_nvme_vs_register
*vs
);
830 int nvme_ctrlr_get_cmbsz(struct spdk_nvme_ctrlr
*ctrlr
, union spdk_nvme_cmbsz_register
*cmbsz
);
831 void nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr
*ctrlr
, const union spdk_nvme_cap_register
*cap
,
832 const union spdk_nvme_vs_register
*vs
);
833 int nvme_qpair_init(struct spdk_nvme_qpair
*qpair
, uint16_t id
,
834 struct spdk_nvme_ctrlr
*ctrlr
,
835 enum spdk_nvme_qprio qprio
,
836 uint32_t num_requests
);
837 void nvme_qpair_deinit(struct spdk_nvme_qpair
*qpair
);
838 void nvme_qpair_enable(struct spdk_nvme_qpair
*qpair
);
839 void nvme_qpair_disable(struct spdk_nvme_qpair
*qpair
);
840 void nvme_qpair_complete_error_reqs(struct spdk_nvme_qpair
*qpair
);
841 int nvme_qpair_submit_request(struct spdk_nvme_qpair
*qpair
,
842 struct nvme_request
*req
);
844 int nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr
*ctrlr
);
845 void nvme_ns_set_identify_data(struct spdk_nvme_ns
*ns
);
846 int nvme_ns_construct(struct spdk_nvme_ns
*ns
, uint32_t id
,
847 struct spdk_nvme_ctrlr
*ctrlr
);
848 void nvme_ns_destruct(struct spdk_nvme_ns
*ns
);
850 int nvme_fabric_ctrlr_set_reg_4(struct spdk_nvme_ctrlr
*ctrlr
, uint32_t offset
, uint32_t value
);
851 int nvme_fabric_ctrlr_set_reg_8(struct spdk_nvme_ctrlr
*ctrlr
, uint32_t offset
, uint64_t value
);
852 int nvme_fabric_ctrlr_get_reg_4(struct spdk_nvme_ctrlr
*ctrlr
, uint32_t offset
, uint32_t *value
);
853 int nvme_fabric_ctrlr_get_reg_8(struct spdk_nvme_ctrlr
*ctrlr
, uint32_t offset
, uint64_t *value
);
854 int nvme_fabric_ctrlr_discover(struct spdk_nvme_ctrlr
*ctrlr
,
855 struct spdk_nvme_probe_ctx
*probe_ctx
);
856 int nvme_fabric_qpair_connect(struct spdk_nvme_qpair
*qpair
, uint32_t num_entries
);
858 static inline struct nvme_request
*
859 nvme_allocate_request(struct spdk_nvme_qpair
*qpair
,
860 const struct nvme_payload
*payload
, uint32_t payload_size
,
861 spdk_nvme_cmd_cb cb_fn
, void *cb_arg
)
863 struct nvme_request
*req
;
865 req
= STAILQ_FIRST(&qpair
->free_req
);
870 STAILQ_REMOVE_HEAD(&qpair
->free_req
, stailq
);
873 * Only memset/zero fields that need it. All other fields
874 * will be initialized appropriately either later in this
875 * function, or before they are needed later in the
876 * submission patch. For example, the children
877 * TAILQ_ENTRY and following members are
878 * only used as part of I/O splitting so we avoid
879 * memsetting them until it is actually needed.
880 * They will be initialized in nvme_request_add_child()
881 * if the request is split.
883 memset(req
, 0, offsetof(struct nvme_request
, payload_size
));
886 req
->cb_arg
= cb_arg
;
887 req
->payload
= *payload
;
888 req
->payload_size
= payload_size
;
889 req
->pid
= g_spdk_nvme_pid
;
890 req
->submit_tick
= 0;
895 static inline struct nvme_request
*
896 nvme_allocate_request_contig(struct spdk_nvme_qpair
*qpair
,
897 void *buffer
, uint32_t payload_size
,
898 spdk_nvme_cmd_cb cb_fn
, void *cb_arg
)
900 struct nvme_payload payload
;
902 payload
= NVME_PAYLOAD_CONTIG(buffer
, NULL
);
904 return nvme_allocate_request(qpair
, &payload
, payload_size
, cb_fn
, cb_arg
);
907 static inline struct nvme_request
*
908 nvme_allocate_request_null(struct spdk_nvme_qpair
*qpair
, spdk_nvme_cmd_cb cb_fn
, void *cb_arg
)
910 return nvme_allocate_request_contig(qpair
, NULL
, 0, cb_fn
, cb_arg
);
913 struct nvme_request
*nvme_allocate_request_user_copy(struct spdk_nvme_qpair
*qpair
,
914 void *buffer
, uint32_t payload_size
,
915 spdk_nvme_cmd_cb cb_fn
, void *cb_arg
, bool host_to_controller
);
918 nvme_complete_request(spdk_nvme_cmd_cb cb_fn
, void *cb_arg
, struct spdk_nvme_qpair
*qpair
,
919 struct nvme_request
*req
, struct spdk_nvme_cpl
*cpl
)
921 struct spdk_nvme_cpl err_cpl
;
922 struct nvme_error_cmd
*cmd
;
924 /* error injection at completion path,
925 * only inject for successful completed commands
927 if (spdk_unlikely(!TAILQ_EMPTY(&qpair
->err_cmd_head
) &&
928 !spdk_nvme_cpl_is_error(cpl
))) {
929 TAILQ_FOREACH(cmd
, &qpair
->err_cmd_head
, link
) {
931 if (cmd
->do_not_submit
) {
935 if ((cmd
->opc
== req
->cmd
.opc
) && cmd
->err_count
) {
938 err_cpl
.status
.sct
= cmd
->status
.sct
;
939 err_cpl
.status
.sc
= cmd
->status
.sc
;
954 nvme_free_request(struct nvme_request
*req
)
957 assert(req
->num_children
== 0);
958 assert(req
->qpair
!= NULL
);
960 STAILQ_INSERT_HEAD(&req
->qpair
->free_req
, req
, stailq
);
964 nvme_qpair_free_request(struct spdk_nvme_qpair
*qpair
, struct nvme_request
*req
)
967 assert(req
->num_children
== 0);
969 STAILQ_INSERT_HEAD(&qpair
->free_req
, req
, stailq
);
972 void nvme_request_remove_child(struct nvme_request
*parent
, struct nvme_request
*child
);
973 int nvme_request_check_timeout(struct nvme_request
*req
, uint16_t cid
,
974 struct spdk_nvme_ctrlr_process
*active_proc
, uint64_t now_tick
);
975 uint64_t nvme_get_quirks(const struct spdk_pci_id
*id
);
977 int nvme_robust_mutex_init_shared(pthread_mutex_t
*mtx
);
978 int nvme_robust_mutex_init_recursive_shared(pthread_mutex_t
*mtx
);
980 const char *spdk_nvme_cpl_get_status_string(const struct spdk_nvme_status
*status
);
981 bool nvme_completion_is_retry(const struct spdk_nvme_cpl
*cpl
);
982 void nvme_qpair_print_command(struct spdk_nvme_qpair
*qpair
, struct spdk_nvme_cmd
*cmd
);
983 void nvme_qpair_print_completion(struct spdk_nvme_qpair
*qpair
, struct spdk_nvme_cpl
*cpl
);
985 struct spdk_nvme_ctrlr
*spdk_nvme_get_ctrlr_by_trid_unsafe(
986 const struct spdk_nvme_transport_id
*trid
);
988 /* Transport specific functions */
989 #define DECLARE_TRANSPORT(name) \
990 struct spdk_nvme_ctrlr *nvme_ ## name ## _ctrlr_construct(const struct spdk_nvme_transport_id *trid, const struct spdk_nvme_ctrlr_opts *opts, \
992 int nvme_ ## name ## _ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr); \
993 int nvme_ ## name ## _ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx, bool direct_connect); \
994 int nvme_ ## name ## _ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr); \
995 int nvme_ ## name ## _ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value); \
996 int nvme_ ## name ## _ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value); \
997 int nvme_ ## name ## _ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value); \
998 int nvme_ ## name ## _ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value); \
999 uint32_t nvme_ ## name ## _ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr); \
1000 uint16_t nvme_ ## name ## _ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr); \
1001 struct spdk_nvme_qpair *nvme_ ## name ## _ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid, const struct spdk_nvme_io_qpair_opts *opts); \
1002 void *nvme_ ## name ## _ctrlr_alloc_cmb_io_buffer(struct spdk_nvme_ctrlr *ctrlr, size_t size); \
1003 int nvme_ ## name ## _ctrlr_free_cmb_io_buffer(struct spdk_nvme_ctrlr *ctrlr, void *buf, size_t size); \
1004 volatile struct spdk_nvme_registers *nvme_ ## name ## _ctrlr_get_registers(struct spdk_nvme_ctrlr *ctrlr); \
1005 int nvme_ ## name ## _ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair); \
1006 int nvme_ ## name ## _ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair); \
1007 void nvme_ ## name ## _ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair); \
1008 void nvme_ ## name ## _qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr); \
1009 int nvme_ ## name ## _qpair_reset(struct spdk_nvme_qpair *qpair); \
1010 int nvme_ ## name ## _qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req); \
1011 int32_t nvme_ ## name ## _qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions); \
1012 void nvme_ ## name ## _admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair); \
1014 DECLARE_TRANSPORT(transport) /* generic transport dispatch functions */
1015 DECLARE_TRANSPORT(pcie
)
1016 DECLARE_TRANSPORT(tcp
)
1017 #ifdef SPDK_CONFIG_RDMA
1018 DECLARE_TRANSPORT(rdma
)
1021 #undef DECLARE_TRANSPORT
1024 * Below ref related functions must be called with the global
1025 * driver lock held for the multi-process condition.
1026 * Within these functions, the per ctrlr ctrlr_lock is also
1027 * acquired for the multi-thread condition.
1029 void nvme_ctrlr_proc_get_ref(struct spdk_nvme_ctrlr
*ctrlr
);
1030 void nvme_ctrlr_proc_put_ref(struct spdk_nvme_ctrlr
*ctrlr
);
1031 int nvme_ctrlr_get_ref_count(struct spdk_nvme_ctrlr
*ctrlr
);
1034 _is_page_aligned(uint64_t address
, uint64_t page_size
)
1036 return (address
& (page_size
- 1)) == 0;
1039 #endif /* __NVME_INTERNAL_H__ */