]> git.proxmox.com Git - mirror_edk2.git/blobdiff - MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c
MdeModulePkg NvmExpressDxe: Fix invalid queue size when creating IO queues
[mirror_edk2.git] / MdeModulePkg / Bus / Pci / NvmExpressDxe / NvmExpressHci.c
index 62eb39a7d6efdd0b0c3106d728108a2133b05de9..a173504cdf4d3a46004df0c62a94816f8e705ed2 100644 (file)
@@ -2,7 +2,7 @@
   NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows\r
   NVM Express specification.\r
 \r
-  Copyright (c) 2013, Intel Corporation. All rights reserved.<BR>\r
+  Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.<BR>\r
   This program and the accompanying materials\r
   are licensed and made available under the terms and conditions of the BSD License\r
   which accompanies this distribution.  The full text of the license may be found at\r
@@ -328,7 +328,7 @@ WriteNvmeAdminSubmissionQueueBaseAddress (
     return Status;\r
   }\r
 \r
-  DEBUG ((EFI_D_INFO, "Asq.Asqb: %lx\n", Asq->Asqb));\r
+  DEBUG ((EFI_D_INFO, "Asq: %lx\n", *Asq));\r
 \r
   return EFI_SUCCESS;\r
 }\r
@@ -408,7 +408,7 @@ WriteNvmeAdminCompletionQueueBaseAddress (
     return Status;\r
   }\r
 \r
-  DEBUG ((EFI_D_INFO, "Acq.Acqb: %lxh\n", Acq->Acqb));\r
+  DEBUG ((EFI_D_INFO, "Acq: %lxh\n", *Acq));\r
 \r
   return EFI_SUCCESS;\r
 }\r
@@ -430,6 +430,8 @@ NvmeDisableController (
   NVME_CC                Cc;\r
   NVME_CSTS              Csts;\r
   EFI_STATUS             Status;\r
+  UINT32                 Index;\r
+  UINT8                  Timeout;\r
 \r
   //\r
   // Read Controller Configuration Register.\r
@@ -450,19 +452,35 @@ NvmeDisableController (
     return Status;\r
   }\r
 \r
-  gBS->Stall(10000);\r
-\r
   //\r
-  // Check if the controller is reset\r
+  // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to transition from 1 to 0 after\r
+  // Cc.Enable transition from 1 to 0. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.\r
   //\r
-  Status = ReadNvmeControllerStatus (Private, &Csts);\r
+  if (Private->Cap.To == 0) {\r
+    Timeout = 1;\r
+  } else {\r
+    Timeout = Private->Cap.To;\r
+  }\r
 \r
-  if (EFI_ERROR(Status)) {\r
-    return Status;\r
+  for(Index = (Timeout * 500); Index != 0; --Index) {\r
+    gBS->Stall(1000);\r
+\r
+    //\r
+    // Check if the controller is initialized\r
+    //\r
+    Status = ReadNvmeControllerStatus (Private, &Csts);\r
+\r
+    if (EFI_ERROR(Status)) {\r
+      return Status;\r
+    }\r
+\r
+    if (Csts.Rdy == 0) {\r
+      break;\r
+    }\r
   }\r
 \r
-  if (Csts.Rdy != 0) {\r
-    return EFI_DEVICE_ERROR;\r
+  if (Index == 0) {\r
+    Status = EFI_DEVICE_ERROR;\r
   }\r
 \r
   DEBUG ((EFI_D_INFO, "NVMe controller is disabled with status [%r].\n", Status));\r
@@ -491,14 +509,15 @@ NvmeEnableController (
   UINT8                  Timeout;\r
 \r
   //\r
-  // Enable the controller\r
+  // Enable the controller.\r
+  // CC.AMS, CC.MPS and CC.CSS are all set to 0.\r
   //\r
   ZeroMem (&Cc, sizeof (NVME_CC));\r
   Cc.En     = 1;\r
   Cc.Iosqes = 6;\r
   Cc.Iocqes = 4;\r
-  Status    = WriteNvmeControllerConfiguration (Private, &Cc);\r
 \r
+  Status = WriteNvmeControllerConfiguration (Private, &Cc);\r
   if (EFI_ERROR(Status)) {\r
     return Status;\r
   }\r
@@ -554,17 +573,16 @@ NvmeIdentifyController (
   IN VOID                               *Buffer\r
   )\r
 {\r
-  NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;\r
-  NVM_EXPRESS_COMMAND                      Command;\r
-  NVM_EXPRESS_RESPONSE                     Response;\r
+  EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;\r
+  EFI_NVM_EXPRESS_COMMAND                  Command;\r
+  EFI_NVM_EXPRESS_COMPLETION               Completion;\r
   EFI_STATUS                               Status;\r
 \r
-  ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
-  ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));\r
-  ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));\r
+  ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
+  ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));\r
+  ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));\r
 \r
-  Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_OPC;\r
-  Command.Cdw0.Cid    = Private->Cid[0]++;\r
+  Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;\r
   //\r
   // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.\r
   // For the Identify command, the Namespace Identifier is only used for the Namespace data structure.\r
@@ -572,11 +590,11 @@ NvmeIdentifyController (
   Command.Nsid        = 0;\r
 \r
   CommandPacket.NvmeCmd        = &Command;\r
-  CommandPacket.NvmeResponse   = &Response;\r
+  CommandPacket.NvmeCompletion = &Completion;\r
   CommandPacket.TransferBuffer = Buffer;\r
   CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA);\r
   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;\r
-  CommandPacket.QueueId        = NVME_ADMIN_QUEUE;\r
+  CommandPacket.QueueType      = NVME_ADMIN_QUEUE;\r
   //\r
   // Set bit 0 (Cns bit) to 1 to identify a controller\r
   //\r
@@ -586,7 +604,6 @@ NvmeIdentifyController (
   Status = Private->Passthru.PassThru (\r
                                &Private->Passthru,\r
                                NVME_CONTROLLER_ID,\r
-                               0,\r
                                &CommandPacket,\r
                                NULL\r
                                );\r
@@ -612,25 +629,24 @@ NvmeIdentifyNamespace (
   IN VOID                              *Buffer\r
   )\r
 {\r
-  NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;\r
-  NVM_EXPRESS_COMMAND                      Command;\r
-  NVM_EXPRESS_RESPONSE                     Response;\r
+  EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;\r
+  EFI_NVM_EXPRESS_COMMAND                  Command;\r
+  EFI_NVM_EXPRESS_COMPLETION               Completion;\r
   EFI_STATUS                               Status;\r
 \r
-  ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
-  ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));\r
-  ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));\r
+  ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
+  ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));\r
+  ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));\r
 \r
-  CommandPacket.NvmeCmd      = &Command;\r
-  CommandPacket.NvmeResponse = &Response;\r
+  CommandPacket.NvmeCmd        = &Command;\r
+  CommandPacket.NvmeCompletion = &Completion;\r
 \r
-  Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_OPC;\r
-  Command.Cdw0.Cid    = Private->Cid[0]++;\r
+  Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;\r
   Command.Nsid        = NamespaceId;\r
   CommandPacket.TransferBuffer = Buffer;\r
   CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA);\r
   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;\r
-  CommandPacket.QueueId        = NVME_ADMIN_QUEUE;\r
+  CommandPacket.QueueType      = NVME_ADMIN_QUEUE;\r
   //\r
   // Set bit 0 (Cns bit) to 1 to identify a namespace\r
   //\r
@@ -640,7 +656,6 @@ NvmeIdentifyNamespace (
   Status = Private->Passthru.PassThru (\r
                                &Private->Passthru,\r
                                NamespaceId,\r
-                               0,\r
                                &CommandPacket,\r
                                NULL\r
                                );\r
@@ -662,40 +677,57 @@ NvmeCreateIoCompletionQueue (
   IN NVME_CONTROLLER_PRIVATE_DATA      *Private\r
   )\r
 {\r
-  NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;\r
-  NVM_EXPRESS_COMMAND                      Command;\r
-  NVM_EXPRESS_RESPONSE                     Response;\r
+  EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;\r
+  EFI_NVM_EXPRESS_COMMAND                  Command;\r
+  EFI_NVM_EXPRESS_COMPLETION               Completion;\r
   EFI_STATUS                               Status;\r
   NVME_ADMIN_CRIOCQ                        CrIoCq;\r
+  UINT32                                   Index;\r
+  UINT16                                   QueueSize;\r
+\r
+  Status = EFI_SUCCESS;\r
+\r
+  for (Index = 1; Index < NVME_MAX_QUEUES; Index++) {\r
+    ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
+    ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));\r
+    ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));\r
+    ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ));\r
+\r
+    CommandPacket.NvmeCmd        = &Command;\r
+    CommandPacket.NvmeCompletion = &Completion;\r
+\r
+    Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_CMD;\r
+    CommandPacket.TransferBuffer = Private->CqBufferPciAddr[Index];\r
+    CommandPacket.TransferLength = EFI_PAGE_SIZE;\r
+    CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;\r
+    CommandPacket.QueueType      = NVME_ADMIN_QUEUE;\r
+\r
+    if (Index == 1) {\r
+      QueueSize = NVME_CCQ_SIZE;\r
+    } else {\r
+      if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) {\r
+        QueueSize = NVME_ASYNC_CCQ_SIZE;\r
+      } else {\r
+        QueueSize = Private->Cap.Mqes;\r
+      }\r
+    }\r
 \r
-  ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
-  ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));\r
-  ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));\r
-  ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ));\r
-\r
-  CommandPacket.NvmeCmd      = &Command;\r
-  CommandPacket.NvmeResponse = &Response;\r
-\r
-  Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_OPC;\r
-  Command.Cdw0.Cid    = Private->Cid[0]++;\r
-  CommandPacket.TransferBuffer = Private->CqBufferPciAddr[1];\r
-  CommandPacket.TransferLength = EFI_PAGE_SIZE;\r
-  CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;\r
-  CommandPacket.QueueId        = NVME_ADMIN_QUEUE;\r
-\r
-  CrIoCq.Qid   = NVME_IO_QUEUE;\r
-  CrIoCq.Qsize = NVME_CCQ_SIZE;\r
-  CrIoCq.Pc    = 1;\r
-  CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));\r
-  CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;\r
-\r
-  Status = Private->Passthru.PassThru (\r
-                               &Private->Passthru,\r
-                               0,\r
-                               0,\r
-                               &CommandPacket,\r
-                               NULL\r
-                               );\r
+    CrIoCq.Qid   = Index;\r
+    CrIoCq.Qsize = QueueSize;\r
+    CrIoCq.Pc    = 1;\r
+    CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));\r
+    CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;\r
+\r
+    Status = Private->Passthru.PassThru (\r
+                                 &Private->Passthru,\r
+                                 0,\r
+                                 &CommandPacket,\r
+                                 NULL\r
+                                 );\r
+    if (EFI_ERROR (Status)) {\r
+      break;\r
+    }\r
+  }\r
 \r
   return Status;\r
 }\r
@@ -714,42 +746,59 @@ NvmeCreateIoSubmissionQueue (
   IN NVME_CONTROLLER_PRIVATE_DATA      *Private\r
   )\r
 {\r
-  NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;\r
-  NVM_EXPRESS_COMMAND                      Command;\r
-  NVM_EXPRESS_RESPONSE                     Response;\r
+  EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;\r
+  EFI_NVM_EXPRESS_COMMAND                  Command;\r
+  EFI_NVM_EXPRESS_COMPLETION               Completion;\r
   EFI_STATUS                               Status;\r
   NVME_ADMIN_CRIOSQ                        CrIoSq;\r
+  UINT32                                   Index;\r
+  UINT16                                   QueueSize;\r
+\r
+  Status = EFI_SUCCESS;\r
+\r
+  for (Index = 1; Index < NVME_MAX_QUEUES; Index++) {\r
+    ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
+    ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));\r
+    ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));\r
+    ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ));\r
+\r
+    CommandPacket.NvmeCmd        = &Command;\r
+    CommandPacket.NvmeCompletion = &Completion;\r
+\r
+    Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_CMD;\r
+    CommandPacket.TransferBuffer = Private->SqBufferPciAddr[Index];\r
+    CommandPacket.TransferLength = EFI_PAGE_SIZE;\r
+    CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;\r
+    CommandPacket.QueueType      = NVME_ADMIN_QUEUE;\r
+\r
+    if (Index == 1) {\r
+      QueueSize = NVME_CSQ_SIZE;\r
+    } else {\r
+      if (Private->Cap.Mqes > NVME_ASYNC_CSQ_SIZE) {\r
+        QueueSize = NVME_ASYNC_CSQ_SIZE;\r
+      } else {\r
+        QueueSize = Private->Cap.Mqes;\r
+      }\r
+    }\r
 \r
-  ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
-  ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));\r
-  ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));\r
-  ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ));\r
-\r
-  CommandPacket.NvmeCmd      = &Command;\r
-  CommandPacket.NvmeResponse = &Response;\r
-\r
-  Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_OPC;\r
-  Command.Cdw0.Cid    = Private->Cid[0]++;\r
-  CommandPacket.TransferBuffer = Private->SqBufferPciAddr[1];\r
-  CommandPacket.TransferLength = EFI_PAGE_SIZE;\r
-  CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;\r
-  CommandPacket.QueueId        = NVME_ADMIN_QUEUE;\r
-\r
-  CrIoSq.Qid   = NVME_IO_QUEUE;\r
-  CrIoSq.Qsize = NVME_CSQ_SIZE;\r
-  CrIoSq.Pc    = 1;\r
-  CrIoSq.Cqid  = NVME_IO_QUEUE;\r
-  CrIoSq.Qprio = 0;\r
-  CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));\r
-  CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;\r
-\r
-  Status = Private->Passthru.PassThru (\r
-                               &Private->Passthru,\r
-                               0,\r
-                               0,\r
-                               &CommandPacket,\r
-                               NULL\r
-                               );\r
+    CrIoSq.Qid   = Index;\r
+    CrIoSq.Qsize = QueueSize;\r
+    CrIoSq.Pc    = 1;\r
+    CrIoSq.Cqid  = Index;\r
+    CrIoSq.Qprio = 0;\r
+    CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));\r
+    CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;\r
+\r
+    Status = Private->Passthru.PassThru (\r
+                                 &Private->Passthru,\r
+                                 0,\r
+                                 &CommandPacket,\r
+                                 NULL\r
+                                 );\r
+    if (EFI_ERROR (Status)) {\r
+      break;\r
+    }\r
+  }\r
 \r
   return Status;\r
 }\r
@@ -774,7 +823,8 @@ NvmeControllerInit (
   NVME_AQA                        Aqa;\r
   NVME_ASQ                        Asq;\r
   NVME_ACQ                        Acq;\r
-\r
+  UINT8                           Sn[21];\r
+  UINT8                           Mn[41];\r
   //\r
   // Save original PCI attributes and enable this controller.\r
   //\r
@@ -798,7 +848,7 @@ NvmeControllerInit (
                     );\r
 \r
   if (!EFI_ERROR (Status)) {\r
-    Supports &= EFI_PCI_DEVICE_ENABLE;\r
+    Supports &= (UINT64)EFI_PCI_DEVICE_ENABLE;\r
     Status    = PciIo->Attributes (\r
                          PciIo,\r
                          EfiPciIoAttributeOperationEnable,\r
@@ -832,6 +882,17 @@ NvmeControllerInit (
 \r
   Private->Cid[0] = 0;\r
   Private->Cid[1] = 0;\r
+  Private->Cid[2] = 0;\r
+  Private->Pt[0]  = 0;\r
+  Private->Pt[1]  = 0;\r
+  Private->Pt[2]  = 0;\r
+  Private->SqTdbl[0].Sqt = 0;\r
+  Private->SqTdbl[1].Sqt = 0;\r
+  Private->SqTdbl[2].Sqt = 0;\r
+  Private->CqHdbl[0].Cqh = 0;\r
+  Private->CqHdbl[1].Cqh = 0;\r
+  Private->CqHdbl[2].Cqh = 0;\r
+  Private->AsyncSqHead   = 0;\r
 \r
   Status = NvmeDisableController (Private);\r
 \r
@@ -850,18 +911,17 @@ NvmeControllerInit (
   //\r
   // Address of admin submission queue.\r
   //\r
-  Asq.Rsvd1 = 0;\r
-  Asq.Asqb  = (UINT64)(UINTN)(Private->BufferPciAddr) >> 12;\r
+  Asq = (UINT64)(UINTN)(Private->BufferPciAddr) & ~0xFFF;\r
 \r
   //\r
   // Address of admin completion queue.\r
   //\r
-  Acq.Rsvd1 = 0;\r
-  Acq.Acqb  = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) >> 12;\r
+  Acq = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) & ~0xFFF;\r
 \r
   //\r
   // Address of I/O submission & completion queue.\r
   //\r
+  ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (6));\r
   Private->SqBuffer[0]        = (NVME_SQ *)(UINTN)(Private->Buffer);\r
   Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr);\r
   Private->CqBuffer[0]        = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE);\r
@@ -870,14 +930,20 @@ NvmeControllerInit (
   Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE);\r
   Private->CqBuffer[1]        = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE);\r
   Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE);\r
+  Private->SqBuffer[2]        = (NVME_SQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE);\r
+  Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE);\r
+  Private->CqBuffer[2]        = (NVME_CQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE);\r
+  Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE);\r
 \r
   DEBUG ((EFI_D_INFO, "Private->Buffer = [%016X]\n", (UINT64)(UINTN)Private->Buffer));\r
-  DEBUG ((EFI_D_INFO, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));\r
-  DEBUG ((EFI_D_INFO, "Admin Completion Queue size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));\r
-  DEBUG ((EFI_D_INFO, "Admin Submission Queue (SqBuffer[0]) = [%016X]\n", Private->SqBuffer[0]));\r
-  DEBUG ((EFI_D_INFO, "Admin Completion Queue (CqBuffer[0]) = [%016X]\n", Private->CqBuffer[0]));\r
-  DEBUG ((EFI_D_INFO, "I/O   Submission Queue (SqBuffer[1]) = [%016X]\n", Private->SqBuffer[1]));\r
-  DEBUG ((EFI_D_INFO, "I/O   Completion Queue (CqBuffer[1]) = [%016X]\n", Private->CqBuffer[1]));\r
+  DEBUG ((EFI_D_INFO, "Admin     Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));\r
+  DEBUG ((EFI_D_INFO, "Admin     Completion Queue size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));\r
+  DEBUG ((EFI_D_INFO, "Admin     Submission Queue (SqBuffer[0]) = [%016X]\n", Private->SqBuffer[0]));\r
+  DEBUG ((EFI_D_INFO, "Admin     Completion Queue (CqBuffer[0]) = [%016X]\n", Private->CqBuffer[0]));\r
+  DEBUG ((EFI_D_INFO, "Sync  I/O Submission Queue (SqBuffer[1]) = [%016X]\n", Private->SqBuffer[1]));\r
+  DEBUG ((EFI_D_INFO, "Sync  I/O Completion Queue (CqBuffer[1]) = [%016X]\n", Private->CqBuffer[1]));\r
+  DEBUG ((EFI_D_INFO, "Async I/O Submission Queue (SqBuffer[2]) = [%016X]\n", Private->SqBuffer[2]));\r
+  DEBUG ((EFI_D_INFO, "Async I/O Completion Queue (CqBuffer[2]) = [%016X]\n", Private->CqBuffer[2]));\r
 \r
   //\r
   // Program admin queue attributes.\r
@@ -911,29 +977,15 @@ NvmeControllerInit (
     return Status;\r
   }\r
 \r
-  //\r
-  // Create one I/O completion queue.\r
-  //\r
-  Status = NvmeCreateIoCompletionQueue (Private);\r
-  if (EFI_ERROR(Status)) {\r
-   return Status;\r
-  }\r
-\r
-  //\r
-  // Create one I/O Submission queue.\r
-  //\r
-  Status = NvmeCreateIoSubmissionQueue (Private);\r
-  if (EFI_ERROR(Status)) {\r
-   return Status;\r
-  }\r
-\r
   //\r
   // Allocate buffer for Identify Controller data\r
   //\r
-  Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof(NVME_ADMIN_CONTROLLER_DATA));\r
-\r
   if (Private->ControllerData == NULL) {\r
-    return EFI_OUT_OF_RESOURCES;\r
+    Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof(NVME_ADMIN_CONTROLLER_DATA));\r
+    \r
+    if (Private->ControllerData == NULL) {\r
+      return EFI_OUT_OF_RESOURCES;\r
+    }\r
   }\r
 \r
   //\r
@@ -950,21 +1002,41 @@ NvmeControllerInit (
   //\r
   // Dump NvmExpress Identify Controller Data\r
   //\r
-  Private->ControllerData->Sn[19] = 0;\r
-  Private->ControllerData->Mn[39] = 0;\r
+  CopyMem (Sn, Private->ControllerData->Sn, sizeof (Private->ControllerData->Sn));\r
+  Sn[20] = 0;\r
+  CopyMem (Mn, Private->ControllerData->Mn, sizeof (Private->ControllerData->Mn));\r
+  Mn[40] = 0;\r
   DEBUG ((EFI_D_INFO, " == NVME IDENTIFY CONTROLLER DATA ==\n"));\r
   DEBUG ((EFI_D_INFO, "    PCI VID   : 0x%x\n", Private->ControllerData->Vid));\r
   DEBUG ((EFI_D_INFO, "    PCI SSVID : 0x%x\n", Private->ControllerData->Ssvid));\r
-  DEBUG ((EFI_D_INFO, "    SN        : %a\n",   (CHAR8 *)(Private->ControllerData->Sn)));\r
-  DEBUG ((EFI_D_INFO, "    MN        : %a\n",   (CHAR8 *)(Private->ControllerData->Mn)));\r
+  DEBUG ((EFI_D_INFO, "    SN        : %a\n",   Sn));\r
+  DEBUG ((EFI_D_INFO, "    MN        : %a\n",   Mn));\r
   DEBUG ((EFI_D_INFO, "    FR        : 0x%x\n", *((UINT64*)Private->ControllerData->Fr)));\r
   DEBUG ((EFI_D_INFO, "    RAB       : 0x%x\n", Private->ControllerData->Rab));\r
-  DEBUG ((EFI_D_INFO, "    IEEE      : 0x%x\n", *(UINT32*)Private->ControllerData->Ieee_oiu));\r
+  DEBUG ((EFI_D_INFO, "    IEEE      : 0x%x\n", *(UINT32*)Private->ControllerData->Ieee_oui));\r
   DEBUG ((EFI_D_INFO, "    AERL      : 0x%x\n", Private->ControllerData->Aerl));\r
   DEBUG ((EFI_D_INFO, "    SQES      : 0x%x\n", Private->ControllerData->Sqes));\r
   DEBUG ((EFI_D_INFO, "    CQES      : 0x%x\n", Private->ControllerData->Cqes));\r
   DEBUG ((EFI_D_INFO, "    NN        : 0x%x\n", Private->ControllerData->Nn));\r
 \r
+  //\r
+  // Create two I/O completion queues.\r
+  // One for blocking I/O, one for non-blocking I/O.\r
+  //\r
+  Status = NvmeCreateIoCompletionQueue (Private);\r
+  if (EFI_ERROR(Status)) {\r
+   return Status;\r
+  }\r
+\r
+  //\r
+  // Create two I/O Submission queues.\r
+  // One for blocking I/O, one for non-blocking I/O.\r
+  //\r
+  Status = NvmeCreateIoSubmissionQueue (Private);\r
+  if (EFI_ERROR(Status)) {\r
+   return Status;\r
+  }\r
+\r
   return Status;\r
 }\r
 \r