ceph/src/spdk/lib/nvme/nvme_qpair.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (c) Intel Corporation.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include "nvme_internal.h"
  35
  36 static void nvme_qpair_fail(struct spdk_nvme_qpair *qpair);
  37
  38 struct nvme_string {
  39         uint16_t        value;
  40         const char      *str;
  41 };
  42
  43 static const struct nvme_string admin_opcode[] = {
  44         { SPDK_NVME_OPC_DELETE_IO_SQ, "DELETE IO SQ" },
  45         { SPDK_NVME_OPC_CREATE_IO_SQ, "CREATE IO SQ" },
  46         { SPDK_NVME_OPC_GET_LOG_PAGE, "GET LOG PAGE" },
  47         { SPDK_NVME_OPC_DELETE_IO_CQ, "DELETE IO CQ" },
  48         { SPDK_NVME_OPC_CREATE_IO_CQ, "CREATE IO CQ" },
  49         { SPDK_NVME_OPC_IDENTIFY, "IDENTIFY" },
  50         { SPDK_NVME_OPC_ABORT, "ABORT" },
  51         { SPDK_NVME_OPC_SET_FEATURES, "SET FEATURES" },
  52         { SPDK_NVME_OPC_GET_FEATURES, "GET FEATURES" },
  53         { SPDK_NVME_OPC_ASYNC_EVENT_REQUEST, "ASYNC EVENT REQUEST" },
  54         { SPDK_NVME_OPC_NS_MANAGEMENT, "NAMESPACE MANAGEMENT" },
  55         { SPDK_NVME_OPC_FIRMWARE_COMMIT, "FIRMWARE COMMIT" },
  56         { SPDK_NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD, "FIRMWARE IMAGE DOWNLOAD" },
  57         { SPDK_NVME_OPC_NS_ATTACHMENT, "NAMESPACE ATTACHMENT" },
  58         { SPDK_NVME_OPC_KEEP_ALIVE, "KEEP ALIVE" },
  59         { SPDK_NVME_OPC_FORMAT_NVM, "FORMAT NVM" },
  60         { SPDK_NVME_OPC_SECURITY_SEND, "SECURITY SEND" },
  61         { SPDK_NVME_OPC_SECURITY_RECEIVE, "SECURITY RECEIVE" },
  62         { 0xFFFF, "ADMIN COMMAND" }
  63 };
  64
  65 static const struct nvme_string io_opcode[] = {
  66         { SPDK_NVME_OPC_FLUSH, "FLUSH" },
  67         { SPDK_NVME_OPC_WRITE, "WRITE" },
  68         { SPDK_NVME_OPC_READ, "READ" },
  69         { SPDK_NVME_OPC_WRITE_UNCORRECTABLE, "WRITE UNCORRECTABLE" },
  70         { SPDK_NVME_OPC_COMPARE, "COMPARE" },
  71         { SPDK_NVME_OPC_WRITE_ZEROES, "WRITE ZEROES" },
  72         { SPDK_NVME_OPC_DATASET_MANAGEMENT, "DATASET MANAGEMENT" },
  73         { SPDK_NVME_OPC_RESERVATION_REGISTER, "RESERVATION REGISTER" },
  74         { SPDK_NVME_OPC_RESERVATION_REPORT, "RESERVATION REPORT" },
  75         { SPDK_NVME_OPC_RESERVATION_ACQUIRE, "RESERVATION ACQUIRE" },
  76         { SPDK_NVME_OPC_RESERVATION_RELEASE, "RESERVATION RELEASE" },
  77         { 0xFFFF, "IO COMMAND" }
  78 };
  79
  80 static const char *
  81 nvme_get_string(const struct nvme_string *strings, uint16_t value)
  82 {
  83         const struct nvme_string *entry;
  84
  85         entry = strings;
  86
  87         while (entry->value != 0xFFFF) {
  88                 if (entry->value == value) {
  89                         return entry->str;
  90                 }
  91                 entry++;
  92         }
  93         return entry->str;
  94 }
  95
  96 static void
  97 nvme_admin_qpair_print_command(struct spdk_nvme_qpair *qpair,
  98                                struct spdk_nvme_cmd *cmd)
  99 {
 100
 101         SPDK_NOTICELOG("%s (%02x) sqid:%d cid:%d nsid:%x "
 102                        "cdw10:%08x cdw11:%08x\n",
 103                        nvme_get_string(admin_opcode, cmd->opc), cmd->opc, qpair->id, cmd->cid,
 104                        cmd->nsid, cmd->cdw10, cmd->cdw11);
 105 }
 106
 107 static void
 108 nvme_io_qpair_print_command(struct spdk_nvme_qpair *qpair,
 109                             struct spdk_nvme_cmd *cmd)
 110 {
 111         assert(qpair != NULL);
 112         assert(cmd != NULL);
 113         switch ((int)cmd->opc) {
 114         case SPDK_NVME_OPC_WRITE:
 115         case SPDK_NVME_OPC_READ:
 116         case SPDK_NVME_OPC_WRITE_UNCORRECTABLE:
 117         case SPDK_NVME_OPC_COMPARE:
 118                 SPDK_NOTICELOG("%s sqid:%d cid:%d nsid:%d "
 119                                "lba:%llu len:%d\n",
 120                                nvme_get_string(io_opcode, cmd->opc), qpair->id, cmd->cid,
 121                                cmd->nsid,
 122                                ((unsigned long long)cmd->cdw11 << 32) + cmd->cdw10,
 123                                (cmd->cdw12 & 0xFFFF) + 1);
 124                 break;
 125         case SPDK_NVME_OPC_FLUSH:
 126         case SPDK_NVME_OPC_DATASET_MANAGEMENT:
 127                 SPDK_NOTICELOG("%s sqid:%d cid:%d nsid:%d\n",
 128                                nvme_get_string(io_opcode, cmd->opc), qpair->id, cmd->cid,
 129                                cmd->nsid);
 130                 break;
 131         default:
 132                 SPDK_NOTICELOG("%s (%02x) sqid:%d cid:%d nsid:%d\n",
 133                                nvme_get_string(io_opcode, cmd->opc), cmd->opc, qpair->id,
 134                                cmd->cid, cmd->nsid);
 135                 break;
 136         }
 137 }
 138
 139 void
 140 nvme_qpair_print_command(struct spdk_nvme_qpair *qpair, struct spdk_nvme_cmd *cmd)
 141 {
 142         assert(qpair != NULL);
 143         assert(cmd != NULL);
 144
 145         if (nvme_qpair_is_admin_queue(qpair)) {
 146                 nvme_admin_qpair_print_command(qpair, cmd);
 147         } else {
 148                 nvme_io_qpair_print_command(qpair, cmd);
 149         }
 150 }
 151
 152 static const struct nvme_string generic_status[] = {
 153         { SPDK_NVME_SC_SUCCESS, "SUCCESS" },
 154         { SPDK_NVME_SC_INVALID_OPCODE, "INVALID OPCODE" },
 155         { SPDK_NVME_SC_INVALID_FIELD, "INVALID FIELD" },
 156         { SPDK_NVME_SC_COMMAND_ID_CONFLICT, "COMMAND ID CONFLICT" },
 157         { SPDK_NVME_SC_DATA_TRANSFER_ERROR, "DATA TRANSFER ERROR" },
 158         { SPDK_NVME_SC_ABORTED_POWER_LOSS, "ABORTED - POWER LOSS" },
 159         { SPDK_NVME_SC_INTERNAL_DEVICE_ERROR, "INTERNAL DEVICE ERROR" },
 160         { SPDK_NVME_SC_ABORTED_BY_REQUEST, "ABORTED - BY REQUEST" },
 161         { SPDK_NVME_SC_ABORTED_SQ_DELETION, "ABORTED - SQ DELETION" },
 162         { SPDK_NVME_SC_ABORTED_FAILED_FUSED, "ABORTED - FAILED FUSED" },
 163         { SPDK_NVME_SC_ABORTED_MISSING_FUSED, "ABORTED - MISSING FUSED" },
 164         { SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT, "INVALID NAMESPACE OR FORMAT" },
 165         { SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR, "COMMAND SEQUENCE ERROR" },
 166         { SPDK_NVME_SC_INVALID_SGL_SEG_DESCRIPTOR, "INVALID SGL SEGMENT DESCRIPTOR" },
 167         { SPDK_NVME_SC_INVALID_NUM_SGL_DESCIRPTORS, "INVALID NUMBER OF SGL DESCRIPTORS" },
 168         { SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID, "DATA SGL LENGTH INVALID" },
 169         { SPDK_NVME_SC_METADATA_SGL_LENGTH_INVALID, "METADATA SGL LENGTH INVALID" },
 170         { SPDK_NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID, "SGL DESCRIPTOR TYPE INVALID" },
 171         { SPDK_NVME_SC_INVALID_CONTROLLER_MEM_BUF, "INVALID CONTROLLER MEMORY BUFFER" },
 172         { SPDK_NVME_SC_INVALID_PRP_OFFSET, "INVALID PRP OFFSET" },
 173         { SPDK_NVME_SC_ATOMIC_WRITE_UNIT_EXCEEDED, "ATOMIC WRITE UNIT EXCEEDED" },
 174         { SPDK_NVME_SC_INVALID_SGL_OFFSET, "INVALID SGL OFFSET" },
 175         { SPDK_NVME_SC_INVALID_SGL_SUBTYPE, "INVALID SGL SUBTYPE" },
 176         { SPDK_NVME_SC_HOSTID_INCONSISTENT_FORMAT, "HOSTID INCONSISTENT FORMAT" },
 177         { SPDK_NVME_SC_KEEP_ALIVE_EXPIRED, "KEEP ALIVE EXPIRED" },
 178         { SPDK_NVME_SC_KEEP_ALIVE_INVALID, "KEEP ALIVE INVALID" },
 179         { SPDK_NVME_SC_LBA_OUT_OF_RANGE, "LBA OUT OF RANGE" },
 180         { SPDK_NVME_SC_CAPACITY_EXCEEDED, "CAPACITY EXCEEDED" },
 181         { SPDK_NVME_SC_NAMESPACE_NOT_READY, "NAMESPACE NOT READY" },
 182         { SPDK_NVME_SC_RESERVATION_CONFLICT, "RESERVATION CONFLICT" },
 183         { SPDK_NVME_SC_FORMAT_IN_PROGRESS, "FORMAT IN PROGRESS" },
 184         { 0xFFFF, "GENERIC" }
 185 };
 186
 187 static const struct nvme_string command_specific_status[] = {
 188         { SPDK_NVME_SC_COMPLETION_QUEUE_INVALID, "INVALID COMPLETION QUEUE" },
 189         { SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER, "INVALID QUEUE IDENTIFIER" },
 190         { SPDK_NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED, "MAX QUEUE SIZE EXCEEDED" },
 191         { SPDK_NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED, "ABORT CMD LIMIT EXCEEDED" },
 192         { SPDK_NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED, "ASYNC LIMIT EXCEEDED" },
 193         { SPDK_NVME_SC_INVALID_FIRMWARE_SLOT, "INVALID FIRMWARE SLOT" },
 194         { SPDK_NVME_SC_INVALID_FIRMWARE_IMAGE, "INVALID FIRMWARE IMAGE" },
 195         { SPDK_NVME_SC_INVALID_INTERRUPT_VECTOR, "INVALID INTERRUPT VECTOR" },
 196         { SPDK_NVME_SC_INVALID_LOG_PAGE, "INVALID LOG PAGE" },
 197         { SPDK_NVME_SC_INVALID_FORMAT, "INVALID FORMAT" },
 198         { SPDK_NVME_SC_FIRMWARE_REQ_CONVENTIONAL_RESET, "FIRMWARE REQUIRES CONVENTIONAL RESET" },
 199         { SPDK_NVME_SC_INVALID_QUEUE_DELETION, "INVALID QUEUE DELETION" },
 200         { SPDK_NVME_SC_FEATURE_ID_NOT_SAVEABLE, "FEATURE ID NOT SAVEABLE" },
 201         { SPDK_NVME_SC_FEATURE_NOT_CHANGEABLE, "FEATURE NOT CHANGEABLE" },
 202         { SPDK_NVME_SC_FEATURE_NOT_NAMESPACE_SPECIFIC, "FEATURE NOT NAMESPACE SPECIFIC" },
 203         { SPDK_NVME_SC_FIRMWARE_REQ_NVM_RESET, "FIRMWARE REQUIRES NVM RESET" },
 204         { SPDK_NVME_SC_FIRMWARE_REQ_RESET, "FIRMWARE REQUIRES RESET" },
 205         { SPDK_NVME_SC_FIRMWARE_REQ_MAX_TIME_VIOLATION, "FIRMWARE REQUIRES MAX TIME VIOLATION" },
 206         { SPDK_NVME_SC_FIRMWARE_ACTIVATION_PROHIBITED, "FIRMWARE ACTIVATION PROHIBITED" },
 207         { SPDK_NVME_SC_OVERLAPPING_RANGE, "OVERLAPPING RANGE" },
 208         { SPDK_NVME_SC_NAMESPACE_INSUFFICIENT_CAPACITY, "NAMESPACE INSUFFICIENT CAPACITY" },
 209         { SPDK_NVME_SC_NAMESPACE_ID_UNAVAILABLE, "NAMESPACE ID UNAVAILABLE" },
 210         { SPDK_NVME_SC_NAMESPACE_ALREADY_ATTACHED, "NAMESPACE ALREADY ATTACHED" },
 211         { SPDK_NVME_SC_NAMESPACE_IS_PRIVATE, "NAMESPACE IS PRIVATE" },
 212         { SPDK_NVME_SC_NAMESPACE_NOT_ATTACHED, "NAMESPACE NOT ATTACHED" },
 213         { SPDK_NVME_SC_THINPROVISIONING_NOT_SUPPORTED, "THINPROVISIONING NOT SUPPORTED" },
 214         { SPDK_NVME_SC_CONTROLLER_LIST_INVALID, "CONTROLLER LIST INVALID" },
 215         { SPDK_NVME_SC_CONFLICTING_ATTRIBUTES, "CONFLICTING ATTRIBUTES" },
 216         { SPDK_NVME_SC_INVALID_PROTECTION_INFO, "INVALID PROTECTION INFO" },
 217         { SPDK_NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE, "WRITE TO RO PAGE" },
 218         { 0xFFFF, "COMMAND SPECIFIC" }
 219 };
 220
 221 static const struct nvme_string media_error_status[] = {
 222         { SPDK_NVME_SC_WRITE_FAULTS, "WRITE FAULTS" },
 223         { SPDK_NVME_SC_UNRECOVERED_READ_ERROR, "UNRECOVERED READ ERROR" },
 224         { SPDK_NVME_SC_GUARD_CHECK_ERROR, "GUARD CHECK ERROR" },
 225         { SPDK_NVME_SC_APPLICATION_TAG_CHECK_ERROR, "APPLICATION TAG CHECK ERROR" },
 226         { SPDK_NVME_SC_REFERENCE_TAG_CHECK_ERROR, "REFERENCE TAG CHECK ERROR" },
 227         { SPDK_NVME_SC_COMPARE_FAILURE, "COMPARE FAILURE" },
 228         { SPDK_NVME_SC_ACCESS_DENIED, "ACCESS DENIED" },
 229         { SPDK_NVME_SC_DEALLOCATED_OR_UNWRITTEN_BLOCK, "DEALLOCATED OR UNWRITTEN BLOCK" },
 230         { 0xFFFF, "MEDIA ERROR" }
 231 };
 232
 233 static const char *
 234 get_status_string(uint16_t sct, uint16_t sc)
 235 {
 236         const struct nvme_string *entry;
 237
 238         switch (sct) {
 239         case SPDK_NVME_SCT_GENERIC:
 240                 entry = generic_status;
 241                 break;
 242         case SPDK_NVME_SCT_COMMAND_SPECIFIC:
 243                 entry = command_specific_status;
 244                 break;
 245         case SPDK_NVME_SCT_MEDIA_ERROR:
 246                 entry = media_error_status;
 247                 break;
 248         case SPDK_NVME_SCT_VENDOR_SPECIFIC:
 249                 return "VENDOR SPECIFIC";
 250         default:
 251                 return "RESERVED";
 252         }
 253
 254         return nvme_get_string(entry, sc);
 255 }
 256
 257 void
 258 nvme_qpair_print_completion(struct spdk_nvme_qpair *qpair,
 259                             struct spdk_nvme_cpl *cpl)
 260 {
 261         SPDK_NOTICELOG("%s (%02x/%02x) sqid:%d cid:%d cdw0:%x sqhd:%04x p:%x m:%x dnr:%x\n",
 262                        get_status_string(cpl->status.sct, cpl->status.sc),
 263                        cpl->status.sct, cpl->status.sc, cpl->sqid, cpl->cid, cpl->cdw0,
 264                        cpl->sqhd, cpl->status.p, cpl->status.m, cpl->status.dnr);
 265 }
 266
 267 bool
 268 nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl)
 269 {
 270         /*
 271          * TODO: spec is not clear how commands that are aborted due
 272          *  to TLER will be marked.  So for now, it seems
 273          *  NAMESPACE_NOT_READY is the only case where we should
 274          *  look at the DNR bit.
 275          */
 276         switch ((int)cpl->status.sct) {
 277         case SPDK_NVME_SCT_GENERIC:
 278                 switch ((int)cpl->status.sc) {
 279                 case SPDK_NVME_SC_NAMESPACE_NOT_READY:
 280                 case SPDK_NVME_SC_FORMAT_IN_PROGRESS:
 281                         if (cpl->status.dnr) {
 282                                 return false;
 283                         } else {
 284                                 return true;
 285                         }
 286                 case SPDK_NVME_SC_INVALID_OPCODE:
 287                 case SPDK_NVME_SC_INVALID_FIELD:
 288                 case SPDK_NVME_SC_COMMAND_ID_CONFLICT:
 289                 case SPDK_NVME_SC_DATA_TRANSFER_ERROR:
 290                 case SPDK_NVME_SC_ABORTED_POWER_LOSS:
 291                 case SPDK_NVME_SC_INTERNAL_DEVICE_ERROR:
 292                 case SPDK_NVME_SC_ABORTED_BY_REQUEST:
 293                 case SPDK_NVME_SC_ABORTED_SQ_DELETION:
 294                 case SPDK_NVME_SC_ABORTED_FAILED_FUSED:
 295                 case SPDK_NVME_SC_ABORTED_MISSING_FUSED:
 296                 case SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT:
 297                 case SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR:
 298                 case SPDK_NVME_SC_LBA_OUT_OF_RANGE:
 299                 case SPDK_NVME_SC_CAPACITY_EXCEEDED:
 300                 default:
 301                         return false;
 302                 }
 303         case SPDK_NVME_SCT_COMMAND_SPECIFIC:
 304         case SPDK_NVME_SCT_MEDIA_ERROR:
 305         case SPDK_NVME_SCT_VENDOR_SPECIFIC:
 306         default:
 307                 return false;
 308         }
 309 }
 310
 311 static void
 312 nvme_qpair_manual_complete_request(struct spdk_nvme_qpair *qpair,
 313                                    struct nvme_request *req, uint32_t sct, uint32_t sc,
 314                                    bool print_on_error)
 315 {
 316         struct spdk_nvme_cpl    cpl;
 317         bool                    error;
 318
 319         memset(&cpl, 0, sizeof(cpl));
 320         cpl.sqid = qpair->id;
 321         cpl.status.sct = sct;
 322         cpl.status.sc = sc;
 323
 324         error = spdk_nvme_cpl_is_error(&cpl);
 325
 326         if (error && print_on_error) {
 327                 nvme_qpair_print_command(qpair, &req->cmd);
 328                 nvme_qpair_print_completion(qpair, &cpl);
 329         }
 330
 331         if (req->cb_fn) {
 332                 req->cb_fn(req->cb_arg, &cpl);
 333         }
 334
 335         nvme_free_request(req);
 336 }
 337
 338 int32_t
 339 spdk_nvme_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions)
 340 {
 341         int32_t ret;
 342
 343         if (qpair->ctrlr->is_failed) {
 344                 nvme_qpair_fail(qpair);
 345                 return 0;
 346         }
 347
 348         qpair->in_completion_context = 1;
 349         ret = nvme_transport_qpair_process_completions(qpair, max_completions);
 350         qpair->in_completion_context = 0;
 351         if (qpair->delete_after_completion_context) {
 352                 /*
 353                  * A request to delete this qpair was made in the context of this completion
 354                  *  routine - so it is safe to delete it now.
 355                  */
 356                 spdk_nvme_ctrlr_free_io_qpair(qpair);
 357         }
 358         return ret;
 359 }
 360
 361 int
 362 nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id,
 363                 struct spdk_nvme_ctrlr *ctrlr,
 364                 enum spdk_nvme_qprio qprio,
 365                 uint32_t num_requests)
 366 {
 367         size_t req_size_padded;
 368         uint32_t i;
 369
 370         qpair->id = id;
 371         qpair->qprio = qprio;
 372
 373         qpair->in_completion_context = 0;
 374         qpair->delete_after_completion_context = 0;
 375
 376         qpair->ctrlr = ctrlr;
 377         qpair->trtype = ctrlr->trid.trtype;
 378
 379         STAILQ_INIT(&qpair->free_req);
 380         STAILQ_INIT(&qpair->queued_req);
 381
 382         req_size_padded = (sizeof(struct nvme_request) + 63) & ~(size_t)63;
 383
 384         qpair->req_buf = spdk_zmalloc(req_size_padded * num_requests, 64, NULL);
 385         if (qpair->req_buf == NULL) {
 386                 return -ENOMEM;
 387         }
 388
 389         for (i = 0; i < num_requests; i++) {
 390                 struct nvme_request *req = qpair->req_buf + i * req_size_padded;
 391
 392                 STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq);
 393         }
 394
 395         return 0;
 396 }
 397
 398 int
 399 nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
 400 {
 401         int                     rc = 0;
 402         struct nvme_request     *child_req, *tmp;
 403         struct spdk_nvme_ctrlr  *ctrlr = qpair->ctrlr;
 404         bool                    child_req_failed = false;
 405
 406         if (ctrlr->is_failed) {
 407                 nvme_free_request(req);
 408                 return -ENXIO;
 409         }
 410
 411         if (req->num_children) {
 412                 /*
 413                  * This is a split (parent) request. Submit all of the children but not the parent
 414                  * request itself, since the parent is the original unsplit request.
 415                  */
 416                 TAILQ_FOREACH_SAFE(child_req, &req->children, child_tailq, tmp) {
 417                         if (!child_req_failed) {
 418                                 rc = nvme_qpair_submit_request(qpair, child_req);
 419                                 if (rc != 0)
 420                                         child_req_failed = true;
 421                         } else { /* free remaining child_reqs since one child_req fails */
 422                                 nvme_request_remove_child(req, child_req);
 423                                 nvme_free_request(child_req);
 424                         }
 425                 }
 426
 427                 return rc;
 428         }
 429
 430         return nvme_transport_qpair_submit_request(qpair, req);
 431 }
 432
 433 static void
 434 _nvme_io_qpair_enable(struct spdk_nvme_qpair *qpair)
 435 {
 436         struct nvme_request             *req;
 437
 438         /* Manually abort each queued I/O. */
 439         while (!STAILQ_EMPTY(&qpair->queued_req)) {
 440                 req = STAILQ_FIRST(&qpair->queued_req);
 441                 STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
 442                 SPDK_ERRLOG("aborting queued i/o\n");
 443                 nvme_qpair_manual_complete_request(qpair, req, SPDK_NVME_SCT_GENERIC,
 444                                                    SPDK_NVME_SC_ABORTED_BY_REQUEST, true);
 445         }
 446 }
 447
 448 void
 449 nvme_qpair_enable(struct spdk_nvme_qpair *qpair)
 450 {
 451         if (nvme_qpair_is_io_queue(qpair)) {
 452                 _nvme_io_qpair_enable(qpair);
 453         }
 454
 455         nvme_transport_qpair_enable(qpair);
 456 }
 457
 458 void
 459 nvme_qpair_disable(struct spdk_nvme_qpair *qpair)
 460 {
 461         nvme_transport_qpair_disable(qpair);
 462 }
 463
 464 static void
 465 nvme_qpair_fail(struct spdk_nvme_qpair *qpair)
 466 {
 467         struct nvme_request             *req;
 468
 469         while (!STAILQ_EMPTY(&qpair->queued_req)) {
 470                 req = STAILQ_FIRST(&qpair->queued_req);
 471                 STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
 472                 SPDK_ERRLOG("failing queued i/o\n");
 473                 nvme_qpair_manual_complete_request(qpair, req, SPDK_NVME_SCT_GENERIC,
 474                                                    SPDK_NVME_SC_ABORTED_BY_REQUEST, true);
 475         }
 476
 477         nvme_transport_qpair_fail(qpair);
 478 }