SecurityPkg/Tcg/Opal/OpalPassword/OpalNvmeMode.c

   1 /** @file
   2   Provide functions to initialize NVME controller and perform NVME commands
   3
   4 Copyright (c) 2016 - 2018, Intel Corporation. All rights reserved.<BR>
   5 This program and the accompanying materials
   6 are licensed and made available under the terms and conditions of the BSD License
   7 which accompanies this distribution.  The full text of the license may be found at
   8 http://opensource.org/licenses/bsd-license.php
   9
  10 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12
  13 **/
  14
  15 #include "OpalPasswordPei.h"
  16
  17
  18 #define ALIGN(v, a)                         (UINTN)((((v) - 1) | ((a) - 1)) + 1)
  19
  20 ///
  21 /// NVME Host controller registers operation
  22 ///
  23 #define NVME_GET_CAP(Nvme, Cap)             NvmeMmioRead  (Cap, Nvme->Nbar + NVME_CAP_OFFSET, sizeof (NVME_CAP))
  24 #define NVME_GET_CC(Nvme, Cc)               NvmeMmioRead  (Cc, Nvme->Nbar + NVME_CC_OFFSET, sizeof (NVME_CC))
  25 #define NVME_SET_CC(Nvme, Cc)               NvmeMmioWrite (Nvme->Nbar + NVME_CC_OFFSET, Cc, sizeof (NVME_CC))
  26 #define NVME_GET_CSTS(Nvme, Csts)           NvmeMmioRead  (Csts, Nvme->Nbar + NVME_CSTS_OFFSET, sizeof (NVME_CSTS))
  27 #define NVME_GET_AQA(Nvme, Aqa)             NvmeMmioRead  (Aqa, Nvme->Nbar + NVME_AQA_OFFSET, sizeof (NVME_AQA))
  28 #define NVME_SET_AQA(Nvme, Aqa)             NvmeMmioWrite (Nvme->Nbar + NVME_AQA_OFFSET, Aqa, sizeof (NVME_AQA))
  29 #define NVME_GET_ASQ(Nvme, Asq)             NvmeMmioRead  (Asq, Nvme->Nbar + NVME_ASQ_OFFSET, sizeof (NVME_ASQ))
  30 #define NVME_SET_ASQ(Nvme, Asq)             NvmeMmioWrite (Nvme->Nbar + NVME_ASQ_OFFSET, Asq, sizeof (NVME_ASQ))
  31 #define NVME_GET_ACQ(Nvme, Acq)             NvmeMmioRead  (Acq, Nvme->Nbar + NVME_ACQ_OFFSET, sizeof (NVME_ACQ))
  32 #define NVME_SET_ACQ(Nvme, Acq)             NvmeMmioWrite (Nvme->Nbar + NVME_ACQ_OFFSET, Acq, sizeof (NVME_ACQ))
  33 #define NVME_GET_VER(Nvme, Ver)             NvmeMmioRead  (Ver, Nvme->Nbar + NVME_VER_OFFSET, sizeof (NVME_VER))
  34 #define NVME_SET_SQTDBL(Nvme, Qid, Sqtdbl)  NvmeMmioWrite (Nvme->Nbar + NVME_SQTDBL_OFFSET(Qid, Nvme->Cap.Dstrd), Sqtdbl, sizeof (NVME_SQTDBL))
  35 #define NVME_SET_CQHDBL(Nvme, Qid, Cqhdbl)  NvmeMmioWrite (Nvme->Nbar + NVME_CQHDBL_OFFSET(Qid, Nvme->Cap.Dstrd), Cqhdbl, sizeof (NVME_CQHDBL))
  36
  37 ///
  38 /// Base memory address
  39 ///
  40 enum {
  41   BASEMEM_CONTROLLER_DATA,
  42   BASEMEM_IDENTIFY_DATA,
  43   BASEMEM_ASQ,
  44   BASEMEM_ACQ,
  45   BASEMEM_SQ,
  46   BASEMEM_CQ,
  47   BASEMEM_PRP,
  48   BASEMEM_SECURITY,
  49   MAX_BASEMEM_COUNT
  50 };
  51
  52 ///
  53 /// All of base memories are 4K(0x1000) alignment
  54 ///
  55 #define NVME_MEM_BASE(Nvme)                 ((UINTN)(Nvme->BaseMem))
  56 #define NVME_CONTROL_DATA_BASE(Nvme)        (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_CONTROLLER_DATA))                        * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  57 #define NVME_NAMESPACE_DATA_BASE(Nvme)      (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_IDENTIFY_DATA))                          * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  58 #define NVME_ASQ_BASE(Nvme)                 (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_ASQ))                                    * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  59 #define NVME_ACQ_BASE(Nvme)                 (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_ACQ))                                    * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  60 #define NVME_SQ_BASE(Nvme, index)           (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_SQ) + ((index)*(NVME_MAX_IO_QUEUES-1)))  * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  61 #define NVME_CQ_BASE(Nvme, index)           (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_CQ) + ((index)*(NVME_MAX_IO_QUEUES-1)))  * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  62 #define NVME_PRP_BASE(Nvme, index)          (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_PRP) + ((index)*NVME_PRP_SIZE))          * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  63 #define NVME_SEC_BASE(Nvme)                 (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_SECURITY))                               * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  64
  65 /**
  66   Transfer MMIO Data to memory.
  67
  68   @param[in,out] MemBuffer - Destination: Memory address
  69   @param[in] MmioAddr      - Source: MMIO address
  70   @param[in] Size          - Size for read
  71
  72   @retval EFI_SUCCESS - MMIO read sucessfully
  73 **/
  74 EFI_STATUS
  75 NvmeMmioRead (
  76   IN OUT VOID *MemBuffer,
  77   IN     UINTN MmioAddr,
  78   IN     UINTN Size
  79   )
  80 {
  81   UINTN  Offset;
  82   UINT8  Data;
  83   UINT8  *Ptr;
  84
  85   // priority has adjusted
  86   switch (Size) {
  87     case 4:
  88       *((UINT32 *)MemBuffer) = MmioRead32 (MmioAddr);
  89       break;
  90
  91     case 8:
  92       *((UINT64 *)MemBuffer) = MmioRead64 (MmioAddr);
  93       break;
  94
  95     case 2:
  96       *((UINT16 *)MemBuffer) = MmioRead16 (MmioAddr);
  97       break;
  98
  99     case 1:
 100       *((UINT8 *)MemBuffer) = MmioRead8 (MmioAddr);
 101       break;
 102
 103     default:
 104       Ptr = (UINT8 *)MemBuffer;
 105       for (Offset = 0; Offset < Size; Offset += 1) {
 106         Data = MmioRead8 (MmioAddr + Offset);
 107         Ptr[Offset] = Data;
 108       }
 109       break;
 110   }
 111
 112   return EFI_SUCCESS;
 113 }
 114
 115 /**
 116   Transfer memory data to MMIO.
 117
 118   @param[in,out] MmioAddr - Destination: MMIO address
 119   @param[in] MemBuffer    - Source: Memory address
 120   @param[in] Size         - Size for write
 121
 122   @retval EFI_SUCCESS - MMIO write sucessfully
 123 **/
 124 EFI_STATUS
 125 NvmeMmioWrite (
 126   IN OUT UINTN MmioAddr,
 127   IN     VOID *MemBuffer,
 128   IN     UINTN Size
 129   )
 130 {
 131   UINTN  Offset;
 132   UINT8  Data;
 133   UINT8  *Ptr;
 134
 135   // priority has adjusted
 136   switch (Size) {
 137     case 4:
 138       MmioWrite32 (MmioAddr, *((UINT32 *)MemBuffer));
 139       break;
 140
 141     case 8:
 142       MmioWrite64 (MmioAddr, *((UINT64 *)MemBuffer));
 143       break;
 144
 145     case 2:
 146       MmioWrite16 (MmioAddr, *((UINT16 *)MemBuffer));
 147       break;
 148
 149     case 1:
 150       MmioWrite8 (MmioAddr, *((UINT8 *)MemBuffer));
 151       break;
 152
 153     default:
 154       Ptr = (UINT8 *)MemBuffer;
 155       for (Offset = 0; Offset < Size; Offset += 1) {
 156         Data = Ptr[Offset];
 157         MmioWrite8 (MmioAddr + Offset, Data);
 158       }
 159       break;
 160   }
 161
 162   return EFI_SUCCESS;
 163 }
 164
 165 /**
 166   Transfer MMIO data to memory.
 167
 168   @param[in,out] MemBuffer - Destination: Memory address
 169   @param[in] MmioAddr      - Source: MMIO address
 170   @param[in] Size          - Size for read
 171
 172   @retval EFI_SUCCESS - MMIO read sucessfully
 173 **/
 174 EFI_STATUS
 175 OpalPciRead (
 176   IN OUT VOID *MemBuffer,
 177   IN     UINTN MmioAddr,
 178   IN     UINTN Size
 179   )
 180 {
 181   UINTN  Offset;
 182   UINT8  Data;
 183   UINT8  *Ptr;
 184
 185   // priority has adjusted
 186   switch (Size) {
 187     case 4:
 188       *((UINT32 *)MemBuffer) = PciRead32 (MmioAddr);
 189       break;
 190
 191     case 2:
 192       *((UINT16 *)MemBuffer) = PciRead16 (MmioAddr);
 193       break;
 194
 195     case 1:
 196       *((UINT8 *)MemBuffer) = PciRead8 (MmioAddr);
 197       break;
 198
 199     default:
 200       Ptr = (UINT8 *)MemBuffer;
 201       for (Offset = 0; Offset < Size; Offset += 1) {
 202         Data = PciRead8 (MmioAddr + Offset);
 203         Ptr[Offset] = Data;
 204       }
 205       break;
 206   }
 207
 208   return EFI_SUCCESS;
 209 }
 210
 211 /**
 212   Transfer memory data to MMIO.
 213
 214   @param[in,out] MmioAddr - Destination: MMIO address
 215   @param[in] MemBuffer    - Source: Memory address
 216   @param[in] Size         - Size for write
 217
 218   @retval EFI_SUCCESS - MMIO write sucessfully
 219 **/
 220 EFI_STATUS
 221 OpalPciWrite (
 222   IN OUT UINTN MmioAddr,
 223   IN     VOID *MemBuffer,
 224   IN     UINTN Size
 225   )
 226 {
 227   UINTN  Offset;
 228   UINT8  Data;
 229   UINT8  *Ptr;
 230
 231   // priority has adjusted
 232   switch (Size) {
 233     case 4:
 234       PciWrite32 (MmioAddr, *((UINT32 *)MemBuffer));
 235       break;
 236
 237     case 2:
 238       PciWrite16 (MmioAddr, *((UINT16 *)MemBuffer));
 239       break;
 240
 241     case 1:
 242       PciWrite8 (MmioAddr, *((UINT8 *)MemBuffer));
 243       break;
 244
 245     default:
 246       Ptr = (UINT8 *)MemBuffer;
 247       for (Offset = 0; Offset < Size; Offset += 1) {
 248         Data = Ptr[Offset];
 249         PciWrite8 (MmioAddr + Offset, Data);
 250       }
 251       break;
 252   }
 253
 254   return EFI_SUCCESS;
 255 }
 256
 257 /**
 258   Get total pages for specific NVME based memory.
 259
 260   @param[in] BaseMemIndex           - The Index of BaseMem (0-based).
 261
 262   @retval - The page count for specific BaseMem Index
 263
 264 **/
 265 UINT32
 266 NvmeGetBaseMemPages (
 267   IN UINTN              BaseMemIndex
 268   )
 269 {
 270   UINT32                Pages;
 271   UINTN                 Index;
 272   UINT32                PageSizeList[8];
 273
 274   PageSizeList[0] = 1;  /* Controller Data */
 275   PageSizeList[1] = 1;  /* Identify Data */
 276   PageSizeList[2] = 1;  /* ASQ */
 277   PageSizeList[3] = 1;  /* ACQ */
 278   PageSizeList[4] = 1;  /* SQs */
 279   PageSizeList[5] = 1;  /* CQs */
 280   PageSizeList[6] = NVME_PRP_SIZE * NVME_CSQ_DEPTH;  /* PRPs */
 281   PageSizeList[7] = 1;  /* Security Commands */
 282
 283   if (BaseMemIndex > MAX_BASEMEM_COUNT) {
 284     ASSERT (FALSE);
 285     return 0;
 286   }
 287
 288   Pages = 0;
 289   for (Index = 0; Index < BaseMemIndex; Index++) {
 290     Pages += PageSizeList[Index];
 291   }
 292
 293   return Pages;
 294 }
 295
 296 /**
 297   Wait for NVME controller status to be ready or not.
 298
 299   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 300   @param[in] WaitReady              - Flag for waitting status ready or not
 301
 302   @return EFI_SUCCESS               - Successfully to wait specific status.
 303   @return others                    - Fail to wait for specific controller status.
 304
 305 **/
 306 STATIC
 307 EFI_STATUS
 308 NvmeWaitController (
 309   IN NVME_CONTEXT       *Nvme,
 310   IN BOOLEAN            WaitReady
 311   )
 312 {
 313   NVME_CSTS              Csts;
 314   EFI_STATUS             Status;
 315   UINT32                 Index;
 316   UINT8                  Timeout;
 317
 318   //
 319   // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
 320   // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
 321   //
 322   if (Nvme->Cap.To == 0) {
 323     Timeout = 1;
 324   } else {
 325     Timeout = Nvme->Cap.To;
 326   }
 327
 328   Status = EFI_SUCCESS;
 329   for(Index = (Timeout * 500); Index != 0; --Index) {
 330     MicroSecondDelay (1000);
 331
 332     //
 333     // Check if the controller is initialized
 334     //
 335     Status = NVME_GET_CSTS (Nvme, &Csts);
 336     if (EFI_ERROR(Status)) {
 337       DEBUG ((DEBUG_ERROR, "NVME_GET_CSTS fail, Status = %r\n", Status));
 338       return Status;
 339     }
 340
 341     if ((BOOLEAN) Csts.Rdy == WaitReady) {
 342       break;
 343     }
 344   }
 345
 346   if (Index == 0) {
 347     Status = EFI_TIMEOUT;
 348   }
 349
 350   return Status;
 351 }
 352
 353 /**
 354   Disable the Nvm Express controller.
 355
 356   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 357
 358   @return EFI_SUCCESS               - Successfully disable the controller.
 359   @return others                    - Fail to disable the controller.
 360
 361 **/
 362 STATIC
 363 EFI_STATUS
 364 NvmeDisableController (
 365   IN NVME_CONTEXT       *Nvme
 366   )
 367 {
 368   NVME_CC                Cc;
 369   NVME_CSTS              Csts;
 370   EFI_STATUS             Status;
 371
 372   Status = NVME_GET_CSTS (Nvme, &Csts);
 373
 374   ///
 375   /// Read Controller Configuration Register.
 376   ///
 377   Status = NVME_GET_CC (Nvme, &Cc);
 378   if (EFI_ERROR(Status)) {
 379     DEBUG ((DEBUG_ERROR, "NVME_GET_CC fail, Status = %r\n", Status));
 380     goto Done;
 381   }
 382
 383   if (Cc.En == 1) {
 384     Cc.En = 0;
 385     ///
 386     /// Disable the controller.
 387     ///
 388     Status = NVME_SET_CC (Nvme, &Cc);
 389     if (EFI_ERROR(Status)) {
 390       DEBUG ((DEBUG_ERROR, "NVME_SET_CC fail, Status = %r\n", Status));
 391       goto Done;
 392     }
 393   }
 394
 395   Status = NvmeWaitController (Nvme, FALSE);
 396   if (EFI_ERROR(Status)) {
 397     DEBUG ((DEBUG_ERROR, "NvmeWaitController fail, Status = %r\n", Status));
 398     goto Done;
 399   }
 400
 401   return EFI_SUCCESS;
 402
 403 Done:
 404   DEBUG ((DEBUG_INFO, "NvmeDisableController fail, Status: %r\n", Status));
 405   return Status;
 406 }
 407
 408 /**
 409   Enable the Nvm Express controller.
 410
 411   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 412
 413   @return EFI_SUCCESS               - Successfully enable the controller.
 414   @return EFI_DEVICE_ERROR          - Fail to enable the controller.
 415   @return EFI_TIMEOUT               - Fail to enable the controller in given time slot.
 416
 417 **/
 418 STATIC
 419 EFI_STATUS
 420 NvmeEnableController (
 421   IN NVME_CONTEXT       *Nvme
 422   )
 423 {
 424   NVME_CC                Cc;
 425   EFI_STATUS             Status;
 426
 427   //
 428   // Enable the controller
 429   //
 430   ZeroMem (&Cc, sizeof (NVME_CC));
 431   Cc.En     = 1;
 432   Cc.Iosqes = 6;
 433   Cc.Iocqes = 4;
 434   Status    = NVME_SET_CC (Nvme, &Cc);
 435   if (EFI_ERROR(Status)) {
 436     DEBUG ((DEBUG_ERROR, "NVME_SET_CC fail, Status = %r\n", Status));
 437     goto Done;
 438   }
 439
 440   Status = NvmeWaitController (Nvme, TRUE);
 441   if (EFI_ERROR(Status)) {
 442     DEBUG ((DEBUG_ERROR, "NvmeWaitController fail, Status = %r\n", Status));
 443     goto Done;
 444   }
 445
 446   return EFI_SUCCESS;
 447
 448 Done:
 449   DEBUG ((DEBUG_INFO, "NvmeEnableController fail, Status: %r\n", Status));
 450   return Status;
 451 }
 452
 453 /**
 454   Shutdown the Nvm Express controller.
 455
 456   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 457
 458   @return EFI_SUCCESS               - Successfully shutdown the controller.
 459   @return EFI_DEVICE_ERROR          - Fail to shutdown the controller.
 460   @return EFI_TIMEOUT               - Fail to shutdown the controller in given time slot.
 461
 462 **/
 463 STATIC
 464 EFI_STATUS
 465 NvmeShutdownController (
 466   IN NVME_CONTEXT       *Nvme
 467   )
 468 {
 469   NVME_CC                Cc;
 470   NVME_CSTS              Csts;
 471   EFI_STATUS             Status;
 472   UINT32                 Index;
 473   UINTN                  Timeout;
 474
 475   Status    = NVME_GET_CC (Nvme, &Cc);
 476   if (EFI_ERROR(Status)) {
 477     DEBUG ((DEBUG_ERROR, "NVME_GET_CC fail, Status = %r\n", Status));
 478     return Status;
 479   }
 480
 481   Cc.Shn     = 1; // Normal shutdown
 482
 483   Status    = NVME_SET_CC (Nvme, &Cc);
 484   if (EFI_ERROR(Status)) {
 485     DEBUG ((DEBUG_ERROR, "NVME_SET_CC fail, Status = %r\n", Status));
 486     return Status;
 487   }
 488
 489   Timeout = NVME_GENERIC_TIMEOUT/1000; // ms
 490   for(Index = (UINT32)(Timeout); Index != 0; --Index) {
 491     MicroSecondDelay (1000);
 492
 493     Status = NVME_GET_CSTS (Nvme, &Csts);
 494     if (EFI_ERROR(Status)) {
 495       DEBUG ((DEBUG_ERROR, "NVME_GET_CSTS fail, Status = %r\n", Status));
 496       return Status;
 497     }
 498
 499     if (Csts.Shst == 2) { // Shutdown processing complete
 500       break;
 501     }
 502   }
 503
 504   if (Index == 0) {
 505     Status = EFI_TIMEOUT;
 506   }
 507
 508   return Status;
 509 }
 510
 511 /**
 512   Check the execution status from a given completion queue entry.
 513
 514   @param[in]     Cq                 - A pointer to the NVME_CQ item.
 515
 516 **/
 517 EFI_STATUS
 518 NvmeCheckCqStatus (
 519   IN NVME_CQ             *Cq
 520   )
 521 {
 522   if (Cq->Sct == 0x0 && Cq->Sc == 0x0) {
 523     return EFI_SUCCESS;
 524   }
 525
 526   DEBUG ((DEBUG_INFO, "Dump NVMe Completion Entry Status from [0x%x]:\n", (UINTN)Cq));
 527   DEBUG ((DEBUG_INFO, "  SQ Identifier : [0x%x], Phase Tag : [%d], Cmd Identifier : [0x%x]\n", Cq->Sqid, Cq->Pt, Cq->Cid));
 528   DEBUG ((DEBUG_INFO, "  NVMe Cmd Execution Result - "));
 529
 530   switch (Cq->Sct) {
 531     case 0x0:
 532       switch (Cq->Sc) {
 533         case 0x0:
 534           DEBUG ((DEBUG_INFO, "Successful Completion\n"));
 535           return EFI_SUCCESS;
 536         case 0x1:
 537           DEBUG ((DEBUG_INFO, "Invalid Command Opcode\n"));
 538           break;
 539         case 0x2:
 540           DEBUG ((DEBUG_INFO, "Invalid Field in Command\n"));
 541           break;
 542         case 0x3:
 543           DEBUG ((DEBUG_INFO, "Command ID Conflict\n"));
 544           break;
 545         case 0x4:
 546           DEBUG ((DEBUG_INFO, "Data Transfer Error\n"));
 547           break;
 548         case 0x5:
 549           DEBUG ((DEBUG_INFO, "Commands Aborted due to Power Loss Notification\n"));
 550           break;
 551         case 0x6:
 552           DEBUG ((DEBUG_INFO, "Internal Device Error\n"));
 553           break;
 554         case 0x7:
 555           DEBUG ((DEBUG_INFO, "Command Abort Requested\n"));
 556           break;
 557         case 0x8:
 558           DEBUG ((DEBUG_INFO, "Command Aborted due to SQ Deletion\n"));
 559           break;
 560         case 0x9:
 561           DEBUG ((DEBUG_INFO, "Command Aborted due to Failed Fused Command\n"));
 562           break;
 563         case 0xA:
 564           DEBUG ((DEBUG_INFO, "Command Aborted due to Missing Fused Command\n"));
 565           break;
 566         case 0xB:
 567           DEBUG ((DEBUG_INFO, "Invalid Namespace or Format\n"));
 568           break;
 569         case 0xC:
 570           DEBUG ((DEBUG_INFO, "Command Sequence Error\n"));
 571           break;
 572         case 0xD:
 573           DEBUG ((DEBUG_INFO, "Invalid SGL Last Segment Descriptor\n"));
 574           break;
 575         case 0xE:
 576           DEBUG ((DEBUG_INFO, "Invalid Number of SGL Descriptors\n"));
 577           break;
 578         case 0xF:
 579           DEBUG ((DEBUG_INFO, "Data SGL Length Invalid\n"));
 580           break;
 581         case 0x10:
 582           DEBUG ((DEBUG_INFO, "Metadata SGL Length Invalid\n"));
 583           break;
 584         case 0x11:
 585           DEBUG ((DEBUG_INFO, "SGL Descriptor Type Invalid\n"));
 586           break;
 587         case 0x80:
 588           DEBUG ((DEBUG_INFO, "LBA Out of Range\n"));
 589           break;
 590         case 0x81:
 591           DEBUG ((DEBUG_INFO, "Capacity Exceeded\n"));
 592           break;
 593         case 0x82:
 594           DEBUG ((DEBUG_INFO, "Namespace Not Ready\n"));
 595           break;
 596         case 0x83:
 597           DEBUG ((DEBUG_INFO, "Reservation Conflict\n"));
 598           break;
 599       }
 600       break;
 601
 602     case 0x1:
 603       switch (Cq->Sc) {
 604         case 0x0:
 605           DEBUG ((DEBUG_INFO, "Completion Queue Invalid\n"));
 606           break;
 607         case 0x1:
 608           DEBUG ((DEBUG_INFO, "Invalid Queue Identifier\n"));
 609           break;
 610         case 0x2:
 611           DEBUG ((DEBUG_INFO, "Maximum Queue Size Exceeded\n"));
 612           break;
 613         case 0x3:
 614           DEBUG ((DEBUG_INFO, "Abort Command Limit Exceeded\n"));
 615           break;
 616         case 0x5:
 617           DEBUG ((DEBUG_INFO, "Asynchronous Event Request Limit Exceeded\n"));
 618           break;
 619         case 0x6:
 620           DEBUG ((DEBUG_INFO, "Invalid Firmware Slot\n"));
 621           break;
 622         case 0x7:
 623           DEBUG ((DEBUG_INFO, "Invalid Firmware Image\n"));
 624           break;
 625         case 0x8:
 626           DEBUG ((DEBUG_INFO, "Invalid Interrupt Vector\n"));
 627           break;
 628         case 0x9:
 629           DEBUG ((DEBUG_INFO, "Invalid Log Page\n"));
 630           break;
 631         case 0xA:
 632           DEBUG ((DEBUG_INFO, "Invalid Format\n"));
 633           break;
 634         case 0xB:
 635           DEBUG ((DEBUG_INFO, "Firmware Application Requires Conventional Reset\n"));
 636           break;
 637         case 0xC:
 638           DEBUG ((DEBUG_INFO, "Invalid Queue Deletion\n"));
 639           break;
 640         case 0xD:
 641           DEBUG ((DEBUG_INFO, "Feature Identifier Not Saveable\n"));
 642           break;
 643         case 0xE:
 644           DEBUG ((DEBUG_INFO, "Feature Not Changeable\n"));
 645           break;
 646         case 0xF:
 647           DEBUG ((DEBUG_INFO, "Feature Not Namespace Specific\n"));
 648           break;
 649         case 0x10:
 650           DEBUG ((DEBUG_INFO, "Firmware Application Requires NVM Subsystem Reset\n"));
 651           break;
 652         case 0x80:
 653           DEBUG ((DEBUG_INFO, "Conflicting Attributes\n"));
 654           break;
 655         case 0x81:
 656           DEBUG ((DEBUG_INFO, "Invalid Protection Information\n"));
 657           break;
 658         case 0x82:
 659           DEBUG ((DEBUG_INFO, "Attempted Write to Read Only Range\n"));
 660           break;
 661       }
 662       break;
 663
 664     case 0x2:
 665       switch (Cq->Sc) {
 666         case 0x80:
 667           DEBUG ((DEBUG_INFO, "Write Fault\n"));
 668           break;
 669         case 0x81:
 670           DEBUG ((DEBUG_INFO, "Unrecovered Read Error\n"));
 671           break;
 672         case 0x82:
 673           DEBUG ((DEBUG_INFO, "End-to-end Guard Check Error\n"));
 674           break;
 675         case 0x83:
 676           DEBUG ((DEBUG_INFO, "End-to-end Application Tag Check Error\n"));
 677           break;
 678         case 0x84:
 679           DEBUG ((DEBUG_INFO, "End-to-end Reference Tag Check Error\n"));
 680           break;
 681         case 0x85:
 682           DEBUG ((DEBUG_INFO, "Compare Failure\n"));
 683           break;
 684         case 0x86:
 685           DEBUG ((DEBUG_INFO, "Access Denied\n"));
 686           break;
 687       }
 688       break;
 689
 690     default:
 691       DEBUG ((DEBUG_INFO, "Unknown error\n"));
 692       break;
 693   }
 694
 695   return EFI_DEVICE_ERROR;
 696 }
 697
 698 /**
 699   Create PRP lists for Data transfer which is larger than 2 memory pages.
 700   Note here we calcuate the number of required PRP lists and allocate them at one time.
 701
 702   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 703   @param[in] SqId                   - The SQ index for this PRP
 704   @param[in] PhysicalAddr           - The physical base address of Data Buffer.
 705   @param[in] Pages                  - The number of pages to be transfered.
 706   @param[out] PrpListHost           - The host base address of PRP lists.
 707   @param[in,out] PrpListNo          - The number of PRP List.
 708
 709   @retval The pointer Value to the first PRP List of the PRP lists.
 710
 711 **/
 712 STATIC
 713 UINT64
 714 NvmeCreatePrpList (
 715   IN     NVME_CONTEXT                 *Nvme,
 716   IN     UINT16                       SqId,
 717   IN     EFI_PHYSICAL_ADDRESS         PhysicalAddr,
 718   IN     UINTN                        Pages,
 719      OUT VOID                         **PrpListHost,
 720   IN OUT UINTN                        *PrpListNo
 721   )
 722 {
 723   UINTN                       PrpEntryNo;
 724   UINT64                      PrpListBase;
 725   UINTN                       PrpListIndex;
 726   UINTN                       PrpEntryIndex;
 727   UINT64                      Remainder;
 728   EFI_PHYSICAL_ADDRESS        PrpListPhyAddr;
 729   UINTN                       Bytes;
 730   UINT8                       *PrpEntry;
 731   EFI_PHYSICAL_ADDRESS        NewPhyAddr;
 732
 733   ///
 734   /// The number of Prp Entry in a memory page.
 735   ///
 736   PrpEntryNo = EFI_PAGE_SIZE / sizeof (UINT64);
 737
 738   ///
 739   /// Calculate total PrpList number.
 740   ///
 741   *PrpListNo = (UINTN) DivU64x64Remainder ((UINT64)Pages, (UINT64)PrpEntryNo, &Remainder);
 742   if (Remainder != 0) {
 743     *PrpListNo += 1;
 744   }
 745
 746   if (*PrpListNo > NVME_PRP_SIZE) {
 747     DEBUG ((DEBUG_INFO, "NvmeCreatePrpList (PhysicalAddr: %lx, Pages: %x) PrpEntryNo: %x\n",
 748       PhysicalAddr, Pages, PrpEntryNo));
 749     DEBUG ((DEBUG_INFO, "*PrpListNo: %x, Remainder: %lx", *PrpListNo, Remainder));
 750     ASSERT (FALSE);
 751   }
 752   *PrpListHost = (VOID *)(UINTN) NVME_PRP_BASE (Nvme, SqId);
 753
 754   Bytes = EFI_PAGES_TO_SIZE (*PrpListNo);
 755   PrpListPhyAddr = (UINT64)(UINTN)(*PrpListHost);
 756
 757   ///
 758   /// Fill all PRP lists except of last one.
 759   ///
 760   ZeroMem (*PrpListHost, Bytes);
 761   for (PrpListIndex = 0; PrpListIndex < *PrpListNo - 1; ++PrpListIndex) {
 762     PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 763
 764     for (PrpEntryIndex = 0; PrpEntryIndex < PrpEntryNo; ++PrpEntryIndex) {
 765       PrpEntry = (UINT8 *)(UINTN) (PrpListBase + PrpEntryIndex * sizeof(UINT64));
 766       if (PrpEntryIndex != PrpEntryNo - 1) {
 767         ///
 768         /// Fill all PRP entries except of last one.
 769         ///
 770         CopyMem (PrpEntry, (VOID *)(UINTN) (&PhysicalAddr), sizeof (UINT64));
 771         PhysicalAddr += EFI_PAGE_SIZE;
 772       } else {
 773         ///
 774         /// Fill last PRP entries with next PRP List pointer.
 775         ///
 776         NewPhyAddr = (PrpListPhyAddr + (PrpListIndex + 1) * EFI_PAGE_SIZE);
 777         CopyMem (PrpEntry, (VOID *)(UINTN) (&NewPhyAddr), sizeof (UINT64));
 778       }
 779     }
 780   }
 781
 782   ///
 783   /// Fill last PRP list.
 784   ///
 785   PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 786   for (PrpEntryIndex = 0; PrpEntryIndex < ((Remainder != 0) ? Remainder : PrpEntryNo); ++PrpEntryIndex) {
 787     PrpEntry = (UINT8 *)(UINTN) (PrpListBase + PrpEntryIndex * sizeof(UINT64));
 788     CopyMem (PrpEntry, (VOID *)(UINTN) (&PhysicalAddr), sizeof (UINT64));
 789
 790     PhysicalAddr += EFI_PAGE_SIZE;
 791   }
 792
 793   return PrpListPhyAddr;
 794 }
 795
 796 /**
 797   Waits until all NVME commands completed.
 798
 799   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 800   @param[in] Qid                    - Queue index
 801
 802   @retval EFI_SUCCESS               - All NVME commands have completed
 803   @retval EFI_TIMEOUT               - Timeout occured
 804   @retval EFI_NOT_READY             - Not all NVME commands have completed
 805   @retval others                    - Error occurred on device side.
 806 **/
 807 EFI_STATUS
 808 NvmeWaitAllComplete (
 809   IN NVME_CONTEXT       *Nvme,
 810   IN UINT8              Qid
 811   )
 812 {
 813   return EFI_SUCCESS;
 814 }
 815
 816 /**
 817   Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function supports
 818   both blocking I/O and nonblocking I/O. The blocking I/O functionality is required, and the nonblocking
 819   I/O functionality is optional.
 820
 821   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 822   @param[in] NamespaceId            - Is a 32 bit Namespace ID to which the Express HCI command packet will be sent.
 823                                       A Value of 0 denotes the NVM Express controller, a Value of all 0FFh in the namespace
 824                                       ID specifies that the command packet should be sent to all valid namespaces.
 825   @param[in] NamespaceUuid          - Is a 64 bit Namespace UUID to which the Express HCI command packet will be sent.
 826                                       A Value of 0 denotes the NVM Express controller, a Value of all 0FFh in the namespace
 827                                       UUID specifies that the command packet should be sent to all valid namespaces.
 828   @param[in,out] Packet             - A pointer to the NVM Express HCI Command Packet to send to the NVMe namespace specified
 829                                       by NamespaceId.
 830
 831   @retval EFI_SUCCESS               - The NVM Express Command Packet was sent by the host. TransferLength bytes were transferred
 832                                       to, or from DataBuffer.
 833   @retval EFI_NOT_READY             - The NVM Express Command Packet could not be sent because the controller is not ready. The caller
 834                                       may retry again later.
 835   @retval EFI_DEVICE_ERROR          - A device error occurred while attempting to send the NVM Express Command Packet.
 836   @retval EFI_INVALID_PARAMETER     - Namespace, or the contents of NVM_EXPRESS_PASS_THRU_COMMAND_PACKET are invalid. The NVM
 837                                       Express Command Packet was not sent, so no additional status information is available.
 838   @retval EFI_UNSUPPORTED           - The command described by the NVM Express Command Packet is not supported by the host adapter.
 839                                       The NVM Express Command Packet was not sent, so no additional status information is available.
 840   @retval EFI_TIMEOUT               - A timeout occurred while waiting for the NVM Express Command Packet to execute.
 841
 842 **/
 843 EFI_STATUS
 844 NvmePassThru (
 845   IN     NVME_CONTEXT                         *Nvme,
 846   IN     UINT32                               NamespaceId,
 847   IN     UINT64                               NamespaceUuid,
 848   IN OUT NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *Packet
 849   )
 850 {
 851   EFI_STATUS                    Status;
 852   NVME_SQ                       *Sq;
 853   NVME_CQ                       *Cq;
 854   UINT8                         Qid;
 855   UINT32                        Bytes;
 856   UINT32                        Offset;
 857   EFI_PHYSICAL_ADDRESS          PhyAddr;
 858   VOID                          *PrpListHost;
 859   UINTN                         PrpListNo;
 860   UINT32                        Timer;
 861   UINTN SqSize;
 862   UINTN CqSize;
 863
 864   ///
 865   /// check the Data fields in Packet parameter.
 866   ///
 867   if ((Nvme == NULL) || (Packet == NULL)) {
 868     DEBUG ((DEBUG_ERROR, "NvmePassThru, invalid parameter: Nvme(%x)/Packet(%x)\n",
 869       (UINTN)Nvme, (UINTN)Packet));
 870     return EFI_INVALID_PARAMETER;
 871   }
 872
 873   if ((Packet->NvmeCmd == NULL) || (Packet->NvmeResponse == NULL)) {
 874     DEBUG ((DEBUG_ERROR, "NvmePassThru, invalid parameter: NvmeCmd(%x)/NvmeResponse(%x)\n",
 875       (UINTN)Packet->NvmeCmd, (UINTN)Packet->NvmeResponse));
 876     return EFI_INVALID_PARAMETER;
 877   }
 878
 879   if (Packet->QueueId != NVME_ADMIN_QUEUE && Packet->QueueId != NVME_IO_QUEUE) {
 880     DEBUG ((DEBUG_ERROR, "NvmePassThru, invalid parameter: QueueId(%x)\n",
 881       Packet->QueueId));
 882     return EFI_INVALID_PARAMETER;
 883   }
 884
 885   PrpListHost = NULL;
 886   PrpListNo   = 0;
 887   Status      = EFI_SUCCESS;
 888
 889   Qid = Packet->QueueId;
 890   Sq  = Nvme->SqBuffer[Qid] + Nvme->SqTdbl[Qid].Sqt;
 891   Cq  = Nvme->CqBuffer[Qid] + Nvme->CqHdbl[Qid].Cqh;
 892   if (Qid == NVME_ADMIN_QUEUE) {
 893     SqSize = NVME_ASQ_SIZE + 1;
 894     CqSize = NVME_ACQ_SIZE + 1;
 895   } else {
 896     SqSize = NVME_CSQ_DEPTH;
 897     CqSize = NVME_CCQ_DEPTH;
 898   }
 899
 900   if (Packet->NvmeCmd->Nsid != NamespaceId) {
 901     DEBUG ((DEBUG_ERROR, "NvmePassThru: Nsid mismatch (%x, %x)\n",
 902       Packet->NvmeCmd->Nsid, NamespaceId));
 903     return EFI_INVALID_PARAMETER;
 904   }
 905
 906   ZeroMem (Sq, sizeof (NVME_SQ));
 907   Sq->Opc  = Packet->NvmeCmd->Cdw0.Opcode;
 908   Sq->Fuse = Packet->NvmeCmd->Cdw0.FusedOperation;
 909   Sq->Cid  = Packet->NvmeCmd->Cdw0.Cid;
 910   Sq->Nsid = Packet->NvmeCmd->Nsid;
 911
 912   ///
 913   /// Currently we only support PRP for Data transfer, SGL is NOT supported.
 914   ///
 915   ASSERT (Sq->Psdt == 0);
 916   if (Sq->Psdt != 0) {
 917     DEBUG ((DEBUG_ERROR, "NvmePassThru: doesn't support SGL mechanism\n"));
 918     return EFI_UNSUPPORTED;
 919   }
 920
 921   Sq->Prp[0] = Packet->TransferBuffer;
 922   Sq->Prp[1] = 0;
 923
 924   if(Packet->MetadataBuffer != (UINT64)(UINTN)NULL) {
 925     Sq->Mptr = Packet->MetadataBuffer;
 926   }
 927
 928   ///
 929   /// If the Buffer Size spans more than two memory pages (page Size as defined in CC.Mps),
 930   /// then build a PRP list in the second PRP submission queue entry.
 931   ///
 932   Offset = ((UINT32)Sq->Prp[0]) & (EFI_PAGE_SIZE - 1);
 933   Bytes  = Packet->TransferLength;
 934
 935   if ((Offset + Bytes) > (EFI_PAGE_SIZE * 2)) {
 936     ///
 937     /// Create PrpList for remaining Data Buffer.
 938     ///
 939     PhyAddr = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
 940     Sq->Prp[1] = NvmeCreatePrpList (Nvme, Nvme->SqTdbl[Qid].Sqt, PhyAddr, EFI_SIZE_TO_PAGES(Offset + Bytes) - 1, &PrpListHost, &PrpListNo);
 941     if (Sq->Prp[1] == 0) {
 942       Status = EFI_OUT_OF_RESOURCES;
 943       DEBUG ((DEBUG_ERROR, "NvmeCreatePrpList fail, Status: %r\n", Status));
 944       goto EXIT;
 945     }
 946
 947   } else if ((Offset + Bytes) > EFI_PAGE_SIZE) {
 948     Sq->Prp[1] = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
 949   }
 950
 951   if(Packet->NvmeCmd->Flags & CDW10_VALID) {
 952     Sq->Payload.Raw.Cdw10 = Packet->NvmeCmd->Cdw10;
 953   }
 954   if(Packet->NvmeCmd->Flags & CDW11_VALID) {
 955     Sq->Payload.Raw.Cdw11 = Packet->NvmeCmd->Cdw11;
 956   }
 957   if(Packet->NvmeCmd->Flags & CDW12_VALID) {
 958     Sq->Payload.Raw.Cdw12 = Packet->NvmeCmd->Cdw12;
 959   }
 960   if(Packet->NvmeCmd->Flags & CDW13_VALID) {
 961     Sq->Payload.Raw.Cdw13 = Packet->NvmeCmd->Cdw13;
 962   }
 963   if(Packet->NvmeCmd->Flags & CDW14_VALID) {
 964     Sq->Payload.Raw.Cdw14 = Packet->NvmeCmd->Cdw14;
 965   }
 966   if(Packet->NvmeCmd->Flags & CDW15_VALID) {
 967     Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
 968   }
 969
 970   ///
 971   /// Ring the submission queue doorbell.
 972   ///
 973   Nvme->SqTdbl[Qid].Sqt++;
 974   if(Nvme->SqTdbl[Qid].Sqt == SqSize) {
 975     Nvme->SqTdbl[Qid].Sqt = 0;
 976   }
 977   Status = NVME_SET_SQTDBL (Nvme, Qid, &Nvme->SqTdbl[Qid]);
 978   if (EFI_ERROR(Status)) {
 979     DEBUG ((DEBUG_ERROR, "NVME_SET_SQTDBL fail, Status: %r\n", Status));
 980     goto EXIT;
 981   }
 982
 983   ///
 984   /// Wait for completion queue to get filled in.
 985   ///
 986   Status = EFI_TIMEOUT;
 987   Timer   = 0;
 988   while (Timer < NVME_CMD_TIMEOUT) {
 989     //DEBUG ((DEBUG_VERBOSE, "Timer: %x, Cq:\n", Timer));
 990     //DumpMem (Cq, sizeof (NVME_CQ));
 991     if (Cq->Pt != Nvme->Pt[Qid]) {
 992       Status = EFI_SUCCESS;
 993       break;
 994     }
 995
 996     MicroSecondDelay (NVME_CMD_WAIT);
 997     Timer += NVME_CMD_WAIT;
 998   }
 999
1000   Nvme->CqHdbl[Qid].Cqh++;
1001   if (Nvme->CqHdbl[Qid].Cqh == CqSize) {
1002     Nvme->CqHdbl[Qid].Cqh = 0;
1003     Nvme->Pt[Qid] ^= 1;
1004   }
1005
1006   ///
1007   /// Copy the Respose Queue entry for this command to the callers response Buffer
1008   ///
1009   CopyMem (Packet->NvmeResponse, Cq, sizeof(NVM_EXPRESS_RESPONSE));
1010
1011   if (!EFI_ERROR(Status)) { // We still need to check CQ status if no timeout error occured
1012     Status = NvmeCheckCqStatus (Cq);
1013   }
1014   NVME_SET_CQHDBL (Nvme, Qid, &Nvme->CqHdbl[Qid]);
1015
1016 EXIT:
1017   return Status;
1018 }
1019
1020 /**
1021   Get identify controller Data.
1022
1023   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1024   @param[in] Buffer                 - The Buffer used to store the identify controller Data.
1025
1026   @return EFI_SUCCESS               - Successfully get the identify controller Data.
1027   @return others                    - Fail to get the identify controller Data.
1028
1029 **/
1030 STATIC
1031 EFI_STATUS
1032 NvmeIdentifyController (
1033   IN NVME_CONTEXT                          *Nvme,
1034   IN VOID                                  *Buffer
1035   )
1036 {
1037   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1038   NVM_EXPRESS_COMMAND                      Command;
1039   NVM_EXPRESS_RESPONSE                     Response;
1040   EFI_STATUS                               Status;
1041
1042   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1043   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1044   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1045
1046   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_OPC;
1047   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1048   //
1049   // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
1050   // For the Identify command, the Namespace Identifier is only used for the Namespace Data structure.
1051   //
1052   Command.Nsid        = 0;
1053
1054   CommandPacket.NvmeCmd        = &Command;
1055   CommandPacket.NvmeResponse   = &Response;
1056   CommandPacket.TransferBuffer = (UINT64)(UINTN)Buffer;
1057   CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA);
1058   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1059   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1060   //
1061   // Set bit 0 (Cns bit) to 1 to identify a controller
1062   //
1063   Command.Cdw10                = 1;
1064   Command.Flags                = CDW10_VALID;
1065
1066   Status = NvmePassThru (
1067               Nvme,
1068               NVME_CONTROLLER_ID,
1069               0,
1070               &CommandPacket
1071               );
1072   if (!EFI_ERROR (Status)) {
1073     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1074   }
1075
1076   return Status;
1077 }
1078
1079 /**
1080   Get specified identify namespace Data.
1081
1082   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1083   @param[in] NamespaceId            - The specified namespace identifier.
1084   @param[in] Buffer                 - The Buffer used to store the identify namespace Data.
1085
1086   @return EFI_SUCCESS               - Successfully get the identify namespace Data.
1087   @return others                    - Fail to get the identify namespace Data.
1088
1089 **/
1090 STATIC
1091 EFI_STATUS
1092 NvmeIdentifyNamespace (
1093   IN NVME_CONTEXT                          *Nvme,
1094   IN UINT32                                NamespaceId,
1095   IN VOID                                  *Buffer
1096   )
1097 {
1098   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1099   NVM_EXPRESS_COMMAND                      Command;
1100   NVM_EXPRESS_RESPONSE                     Response;
1101   EFI_STATUS                               Status;
1102
1103   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1104   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1105   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1106
1107   CommandPacket.NvmeCmd      = &Command;
1108   CommandPacket.NvmeResponse = &Response;
1109
1110   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_OPC;
1111   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1112   Command.Nsid        = NamespaceId;
1113   CommandPacket.TransferBuffer = (UINT64)(UINTN)Buffer;
1114   CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA);
1115   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1116   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1117   //
1118   // Set bit 0 (Cns bit) to 1 to identify a namespace
1119   //
1120   CommandPacket.NvmeCmd->Cdw10 = 0;
1121   CommandPacket.NvmeCmd->Flags = CDW10_VALID;
1122
1123   Status = NvmePassThru (
1124               Nvme,
1125               NamespaceId,
1126               0,
1127               &CommandPacket
1128               );
1129   if (!EFI_ERROR (Status)) {
1130     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1131   }
1132
1133   return Status;
1134 }
1135
1136 /**
1137   Get Block Size for specific namespace of NVME.
1138
1139   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1140
1141   @return                           - Block Size in bytes
1142
1143 **/
1144 STATIC
1145 UINT32
1146 NvmeGetBlockSize (
1147   IN NVME_CONTEXT       *Nvme
1148   )
1149 {
1150   UINT32                BlockSize;
1151   UINT32                Lbads;
1152   UINT32                Flbas;
1153   UINT32                LbaFmtIdx;
1154
1155   Flbas     = Nvme->NamespaceData->Flbas;
1156   LbaFmtIdx = Flbas & 3;
1157   Lbads     = Nvme->NamespaceData->LbaFormat[LbaFmtIdx].Lbads;
1158
1159   BlockSize = (UINT32)1 << Lbads;
1160   return BlockSize;
1161 }
1162
1163 /**
1164   Get last LBA for specific namespace of NVME.
1165
1166   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1167
1168   @return                           - Last LBA address
1169
1170 **/
1171 STATIC
1172 EFI_LBA
1173 NvmeGetLastLba (
1174   IN NVME_CONTEXT       *Nvme
1175   )
1176 {
1177   EFI_LBA               LastBlock;
1178   LastBlock = Nvme->NamespaceData->Nsze - 1;
1179   return LastBlock;
1180 }
1181
1182 /**
1183   Create io completion queue.
1184
1185   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1186
1187   @return EFI_SUCCESS               - Successfully create io completion queue.
1188   @return others                    - Fail to create io completion queue.
1189
1190 **/
1191 STATIC
1192 EFI_STATUS
1193 NvmeCreateIoCompletionQueue (
1194   IN     NVME_CONTEXT                      *Nvme
1195   )
1196 {
1197   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1198   NVM_EXPRESS_COMMAND                      Command;
1199   NVM_EXPRESS_RESPONSE                     Response;
1200   EFI_STATUS                               Status;
1201   NVME_ADMIN_CRIOCQ                        CrIoCq;
1202
1203   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1204   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1205   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1206   ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ));
1207
1208   CommandPacket.NvmeCmd      = &Command;
1209   CommandPacket.NvmeResponse = &Response;
1210
1211   Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_OPC;
1212   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1213   CommandPacket.TransferBuffer = (UINT64)(UINTN)Nvme->CqBuffer[NVME_IO_QUEUE];
1214   CommandPacket.TransferLength = EFI_PAGE_SIZE;
1215   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1216   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1217
1218   CrIoCq.Qid   = NVME_IO_QUEUE;
1219   CrIoCq.Qsize = NVME_CCQ_SIZE;
1220   CrIoCq.Pc    = 1;
1221   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
1222   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1223
1224   Status = NvmePassThru (
1225               Nvme,
1226               NVME_CONTROLLER_ID,
1227               0,
1228               &CommandPacket
1229               );
1230   if (!EFI_ERROR (Status)) {
1231     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1232   }
1233
1234   return Status;
1235 }
1236
1237 /**
1238   Create io submission queue.
1239
1240   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1241
1242   @return EFI_SUCCESS               - Successfully create io submission queue.
1243   @return others                    - Fail to create io submission queue.
1244
1245 **/
1246 STATIC
1247 EFI_STATUS
1248 NvmeCreateIoSubmissionQueue (
1249   IN NVME_CONTEXT                          *Nvme
1250   )
1251 {
1252   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1253   NVM_EXPRESS_COMMAND                      Command;
1254   NVM_EXPRESS_RESPONSE                     Response;
1255   EFI_STATUS                               Status;
1256   NVME_ADMIN_CRIOSQ                        CrIoSq;
1257
1258   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1259   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1260   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1261   ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ));
1262
1263   CommandPacket.NvmeCmd      = &Command;
1264   CommandPacket.NvmeResponse = &Response;
1265
1266   Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_OPC;
1267   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1268   CommandPacket.TransferBuffer = (UINT64)(UINTN)Nvme->SqBuffer[NVME_IO_QUEUE];
1269   CommandPacket.TransferLength = EFI_PAGE_SIZE;
1270   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1271   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1272
1273   CrIoSq.Qid   = NVME_IO_QUEUE;
1274   CrIoSq.Qsize = NVME_CSQ_SIZE;
1275   CrIoSq.Pc    = 1;
1276   CrIoSq.Cqid  = NVME_IO_QUEUE;
1277   CrIoSq.Qprio = 0;
1278   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
1279   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1280
1281   Status = NvmePassThru (
1282               Nvme,
1283               NVME_CONTROLLER_ID,
1284               0,
1285               &CommandPacket
1286               );
1287   if (!EFI_ERROR (Status)) {
1288     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1289   }
1290
1291   return Status;
1292 }
1293
1294 /**
1295   Security send and receive commands.
1296
1297   @param[in]     Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1298   @param[in]     SendCommand            - The flag to indicate the command type, TRUE for Send command and FALSE for receive command
1299   @param[in]     SecurityProtocol       - Security Protocol
1300   @param[in]     SpSpecific             - Security Protocol Specific
1301   @param[in]     TransferLength         - Transfer Length of Buffer (in bytes) - always a multiple of 512
1302   @param[in,out] TransferBuffer         - Address of Data to transfer
1303
1304   @return EFI_SUCCESS               - Successfully create io submission queue.
1305   @return others                    - Fail to send/receive commands.
1306
1307 **/
1308 EFI_STATUS
1309 NvmeSecuritySendReceive (
1310   IN NVME_CONTEXT                          *Nvme,
1311   IN BOOLEAN                               SendCommand,
1312   IN UINT8                                 SecurityProtocol,
1313   IN UINT16                                SpSpecific,
1314   IN UINTN                                 TransferLength,
1315   IN OUT VOID                              *TransferBuffer
1316   )
1317 {
1318   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1319   NVM_EXPRESS_COMMAND                      Command;
1320   NVM_EXPRESS_RESPONSE                     Response;
1321   EFI_STATUS                               Status;
1322   NVME_ADMIN_SECSEND                       SecSend;
1323   OACS                                     *Oacs;
1324   UINT8                                    Opcode;
1325   VOID*                                    *SecBuff;
1326
1327   Oacs = (OACS *)&Nvme->ControllerData->Oacs;
1328
1329   //
1330   // Verify security bit for Security Send/Receive commands
1331   //
1332   if (Oacs->Security == 0) {
1333     DEBUG ((DEBUG_ERROR, "Security command doesn't support.\n"));
1334     return EFI_NOT_READY;
1335   }
1336
1337   SecBuff = (VOID *)(UINTN) NVME_SEC_BASE (Nvme);
1338
1339   //
1340   // Actions for sending security command
1341   //
1342   if (SendCommand) {
1343     CopyMem (SecBuff, TransferBuffer, TransferLength);
1344   }
1345
1346   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1347   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1348   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1349   ZeroMem (&SecSend, sizeof(NVME_ADMIN_SECSEND));
1350
1351   CommandPacket.NvmeCmd      = &Command;
1352   CommandPacket.NvmeResponse = &Response;
1353
1354   Opcode = (UINT8)(SendCommand ? NVME_ADMIN_SECURITY_SEND_OPC : NVME_ADMIN_SECURITY_RECV_OPC);
1355   Command.Cdw0.Opcode = Opcode;
1356   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1357   CommandPacket.TransferBuffer = (UINT64)(UINTN)SecBuff;
1358   CommandPacket.TransferLength = (UINT32)TransferLength;
1359   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1360   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1361
1362   SecSend.Spsp = SpSpecific;
1363   SecSend.Secp = SecurityProtocol;
1364   SecSend.Tl   = (UINT32)TransferLength;
1365
1366   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &SecSend, sizeof (NVME_ADMIN_SECSEND));
1367   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1368
1369   Status = NvmePassThru (
1370               Nvme,
1371               NVME_CONTROLLER_ID,
1372               0,
1373               &CommandPacket
1374               );
1375   if (!EFI_ERROR (Status)) {
1376     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1377   }
1378
1379   //
1380   // Actions for receiving security command
1381   //
1382   if (!SendCommand) {
1383     CopyMem (TransferBuffer, SecBuff, TransferLength);
1384   }
1385
1386   return Status;
1387 }
1388
1389 /**
1390   Destroy io completion queue.
1391
1392   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1393
1394   @return EFI_SUCCESS               - Successfully destroy io completion queue.
1395   @return others                    - Fail to destroy io completion queue.
1396
1397 **/
1398 STATIC
1399 EFI_STATUS
1400 NvmeDestroyIoCompletionQueue (
1401   IN     NVME_CONTEXT                      *Nvme
1402   )
1403 {
1404   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1405   NVM_EXPRESS_COMMAND                      Command;
1406   NVM_EXPRESS_RESPONSE                     Response;
1407   EFI_STATUS                               Status;
1408   NVME_ADMIN_DEIOCQ                        DelIoCq;
1409
1410   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1411   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1412   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1413   ZeroMem (&DelIoCq, sizeof(NVME_ADMIN_DEIOCQ));
1414
1415   CommandPacket.NvmeCmd      = &Command;
1416   CommandPacket.NvmeResponse = &Response;
1417
1418   Command.Cdw0.Opcode = NVME_ADMIN_DELIOCQ_OPC;
1419   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1420   CommandPacket.TransferBuffer = (UINT64)(UINTN)Nvme->CqBuffer[NVME_IO_QUEUE];
1421   CommandPacket.TransferLength = EFI_PAGE_SIZE;
1422   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1423   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1424
1425   DelIoCq.Qid   = NVME_IO_QUEUE;
1426   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &DelIoCq, sizeof (NVME_ADMIN_DEIOCQ));
1427   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1428
1429   Status = NvmePassThru (
1430               Nvme,
1431               NVME_CONTROLLER_ID,
1432               0,
1433               &CommandPacket
1434               );
1435   if (!EFI_ERROR (Status)) {
1436     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1437   }
1438
1439   return Status;
1440 }
1441
1442 /**
1443   Destroy io submission queue.
1444
1445   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1446
1447   @return EFI_SUCCESS               - Successfully destroy io submission queue.
1448   @return others                    - Fail to destroy io submission queue.
1449
1450 **/
1451 STATIC
1452 EFI_STATUS
1453 NvmeDestroyIoSubmissionQueue (
1454   IN NVME_CONTEXT                          *Nvme
1455   )
1456 {
1457   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1458   NVM_EXPRESS_COMMAND                      Command;
1459   NVM_EXPRESS_RESPONSE                     Response;
1460   EFI_STATUS                               Status;
1461   NVME_ADMIN_DEIOSQ                        DelIoSq;
1462
1463   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1464   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1465   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1466   ZeroMem (&DelIoSq, sizeof(NVME_ADMIN_DEIOSQ));
1467
1468   CommandPacket.NvmeCmd      = &Command;
1469   CommandPacket.NvmeResponse = &Response;
1470
1471   Command.Cdw0.Opcode = NVME_ADMIN_DELIOSQ_OPC;
1472   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1473   CommandPacket.TransferBuffer = (UINT64)(UINTN)Nvme->SqBuffer[NVME_IO_QUEUE];
1474   CommandPacket.TransferLength = EFI_PAGE_SIZE;
1475   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1476   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1477
1478   DelIoSq.Qid   = NVME_IO_QUEUE;
1479   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &DelIoSq, sizeof (NVME_ADMIN_DEIOSQ));
1480   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1481
1482   Status = NvmePassThru (
1483               Nvme,
1484               NVME_CONTROLLER_ID,
1485               0,
1486               &CommandPacket
1487               );
1488   if (!EFI_ERROR (Status)) {
1489     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1490   }
1491
1492   return Status;
1493 }
1494
1495 /**
1496   Allocate transfer-related Data struct which is used at Nvme.
1497
1498   @param[in, out] Nvme          The pointer to the NVME_CONTEXT Data structure.
1499
1500   @retval EFI_OUT_OF_RESOURCE   No enough resource.
1501   @retval EFI_SUCCESS           Successful to allocate resource.
1502
1503 **/
1504 EFI_STATUS
1505 EFIAPI
1506 NvmeAllocateResource (
1507   IN OUT NVME_CONTEXT       *Nvme
1508   )
1509 {
1510   EFI_STATUS                Status;
1511   EFI_PHYSICAL_ADDRESS      DeviceAddress;
1512   VOID                      *Base;
1513   VOID                      *Mapping;
1514
1515   //
1516   // Allocate resources for DMA.
1517   //
1518   Status = IoMmuAllocateBuffer (
1519              EFI_SIZE_TO_PAGES (NVME_MEM_MAX_SIZE),
1520              &Base,
1521              &DeviceAddress,
1522              &Mapping
1523              );
1524   if (EFI_ERROR (Status)) {
1525     return EFI_OUT_OF_RESOURCES;
1526   }
1527   ASSERT (DeviceAddress == ((EFI_PHYSICAL_ADDRESS) (UINTN) Base));
1528   Nvme->BaseMemMapping = Mapping;
1529   Nvme->BaseMem = Base;
1530   ZeroMem (Nvme->BaseMem, EFI_PAGE_SIZE * EFI_SIZE_TO_PAGES (NVME_MEM_MAX_SIZE));
1531
1532   DEBUG ((
1533     DEBUG_INFO,
1534     "%a() NvmeContext 0x%x\n",
1535     __FUNCTION__,
1536     Nvme->BaseMem
1537     ));
1538
1539   return EFI_SUCCESS;
1540 }
1541
1542 /**
1543   Free allocated transfer-related Data struct which is used at NVMe.
1544
1545   @param[in, out] Nvme          The pointer to the NVME_CONTEXT Data structure.
1546
1547 **/
1548 VOID
1549 EFIAPI
1550 NvmeFreeResource (
1551   IN OUT NVME_CONTEXT       *Nvme
1552   )
1553 {
1554   if (Nvme->BaseMem != NULL) {
1555     IoMmuFreeBuffer (
1556        EFI_SIZE_TO_PAGES (NVME_MEM_MAX_SIZE),
1557        Nvme->BaseMem,
1558        Nvme->BaseMemMapping
1559        );
1560     Nvme->BaseMem = NULL;
1561   }
1562 }
1563
1564 /**
1565   Initialize the Nvm Express controller.
1566
1567   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1568
1569   @retval EFI_SUCCESS               - The NVM Express Controller is initialized successfully.
1570   @retval Others                    - A device error occurred while initializing the controller.
1571
1572 **/
1573 EFI_STATUS
1574 NvmeControllerInit (
1575   IN NVME_CONTEXT       *Nvme
1576   )
1577 {
1578   EFI_STATUS            Status;
1579   NVME_AQA              Aqa;
1580   NVME_ASQ              Asq;
1581   NVME_ACQ              Acq;
1582   NVME_VER              Ver;
1583
1584   UINT32                MlBAR;
1585   UINT32                MuBAR;
1586
1587   ///
1588   /// Update PCIE BAR0/1 for NVME device
1589   ///
1590   MlBAR = Nvme->Nbar;
1591   MuBAR = 0;
1592   PciWrite32 (Nvme->PciBase + 0x10, MlBAR); // MLBAR (BAR0)
1593   PciWrite32 (Nvme->PciBase + 0x14, MuBAR); // MUBAR (BAR1)
1594
1595   ///
1596   /// Enable PCIE decode
1597   ///
1598   PciWrite8 (Nvme->PciBase + NVME_PCIE_PCICMD, 0x6);
1599
1600   // Version
1601   NVME_GET_VER (Nvme, &Ver);
1602   if (!(Ver.Mjr == 0x0001) && (Ver.Mnr == 0x0000)) {
1603     DEBUG ((DEBUG_INFO, "\n!!!\n!!! NVME Version mismatch for the implementation !!!\n!!!\n"));
1604   }
1605
1606   ///
1607   /// Read the Controller Capabilities register and verify that the NVM command set is supported
1608   ///
1609   Status = NVME_GET_CAP (Nvme, &Nvme->Cap);
1610   if (EFI_ERROR (Status)) {
1611     DEBUG ((DEBUG_ERROR, "NVME_GET_CAP fail, Status: %r\n", Status));
1612     goto Done;
1613   }
1614
1615   if (Nvme->Cap.Css != 0x01) {
1616     DEBUG ((DEBUG_ERROR, "NvmeControllerInit fail: the controller doesn't support NVMe command set\n"));
1617     Status = EFI_UNSUPPORTED;
1618     goto Done;
1619   }
1620
1621   ///
1622   /// Currently the driver only supports 4k page Size.
1623   ///
1624   if ((Nvme->Cap.Mpsmin + 12) > EFI_PAGE_SHIFT) {
1625     DEBUG ((DEBUG_ERROR, "NvmeControllerInit fail: only supports 4k page Size\n"));
1626     ASSERT (FALSE);
1627     Status = EFI_UNSUPPORTED;
1628     goto Done;
1629   }
1630
1631   Nvme->Cid[0] = 0;
1632   Nvme->Cid[1] = 0;
1633
1634   Nvme->Pt[0]  = 0;
1635   Nvme->Pt[1]  = 0;
1636
1637   ZeroMem ((VOID *)(UINTN)(&(Nvme->SqTdbl[0])), sizeof (NVME_SQTDBL) * NVME_MAX_IO_QUEUES);
1638   ZeroMem ((VOID *)(UINTN)(&(Nvme->CqHdbl[0])), sizeof (NVME_CQHDBL) * NVME_MAX_IO_QUEUES);
1639
1640   ZeroMem (Nvme->BaseMem, NVME_MEM_MAX_SIZE);
1641
1642   Status = NvmeDisableController (Nvme);
1643   if (EFI_ERROR(Status)) {
1644     DEBUG ((DEBUG_ERROR, "NvmeDisableController fail, Status: %r\n", Status));
1645     goto Done;
1646   }
1647
1648   ///
1649   /// set number of entries admin submission & completion queues.
1650   ///
1651   Aqa.Asqs  = NVME_ASQ_SIZE;
1652   Aqa.Rsvd1 = 0;
1653   Aqa.Acqs  = NVME_ACQ_SIZE;
1654   Aqa.Rsvd2 = 0;
1655
1656   ///
1657   /// Address of admin submission queue.
1658   ///
1659   Asq = (UINT64)(UINTN)(NVME_ASQ_BASE (Nvme) & ~0xFFF);
1660
1661   ///
1662   /// Address of admin completion queue.
1663   ///
1664   Acq = (UINT64)(UINTN)(NVME_ACQ_BASE (Nvme) & ~0xFFF);
1665
1666   ///
1667   /// Address of I/O submission & completion queue.
1668   ///
1669   Nvme->SqBuffer[0] = (NVME_SQ *)(UINTN)NVME_ASQ_BASE (Nvme);   // NVME_ADMIN_QUEUE
1670   Nvme->CqBuffer[0] = (NVME_CQ *)(UINTN)NVME_ACQ_BASE (Nvme);   // NVME_ADMIN_QUEUE
1671   Nvme->SqBuffer[1] = (NVME_SQ *)(UINTN)NVME_SQ_BASE (Nvme, 0); // NVME_IO_QUEUE
1672   Nvme->CqBuffer[1] = (NVME_CQ *)(UINTN)NVME_CQ_BASE (Nvme, 0); // NVME_IO_QUEUE
1673
1674   DEBUG ((DEBUG_INFO, "Admin Submission Queue Size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));
1675   DEBUG ((DEBUG_INFO, "Admin Completion Queue Size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));
1676   DEBUG ((DEBUG_INFO, "Admin Submission Queue (SqBuffer[0]) =   [%08X]\n", Nvme->SqBuffer[0]));
1677   DEBUG ((DEBUG_INFO, "Admin Completion Queue (CqBuffer[0]) =   [%08X]\n", Nvme->CqBuffer[0]));
1678   DEBUG ((DEBUG_INFO, "I/O   Submission Queue (SqBuffer[1]) =   [%08X]\n", Nvme->SqBuffer[1]));
1679   DEBUG ((DEBUG_INFO, "I/O   Completion Queue (CqBuffer[1]) =   [%08X]\n", Nvme->CqBuffer[1]));
1680
1681   ///
1682   /// Program admin queue attributes.
1683   ///
1684   Status = NVME_SET_AQA (Nvme, &Aqa);
1685   if (EFI_ERROR(Status)) {
1686     goto Done;
1687   }
1688
1689   ///
1690   /// Program admin submission queue address.
1691   ///
1692   Status = NVME_SET_ASQ (Nvme, &Asq);
1693   if (EFI_ERROR(Status)) {
1694     goto Done;
1695   }
1696
1697   ///
1698   /// Program admin completion queue address.
1699   ///
1700   Status = NVME_SET_ACQ (Nvme, &Acq);
1701   if (EFI_ERROR(Status)) {
1702     goto Done;
1703   }
1704
1705   Status = NvmeEnableController (Nvme);
1706   if (EFI_ERROR(Status)) {
1707     goto Done;
1708   }
1709
1710   ///
1711   /// Create one I/O completion queue.
1712   ///
1713   Status = NvmeCreateIoCompletionQueue (Nvme);
1714   if (EFI_ERROR(Status)) {
1715     goto Done;
1716   }
1717
1718   ///
1719   /// Create one I/O Submission queue.
1720   ///
1721   Status = NvmeCreateIoSubmissionQueue (Nvme);
1722   if (EFI_ERROR(Status)) {
1723     goto Done;
1724   }
1725
1726   ///
1727   /// Get current Identify Controller Data
1728   ///
1729   Nvme->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)(UINTN) NVME_CONTROL_DATA_BASE (Nvme);
1730   Status = NvmeIdentifyController (Nvme, Nvme->ControllerData);
1731   if (EFI_ERROR(Status)) {
1732     goto Done;
1733   }
1734
1735   ///
1736   /// Dump NvmExpress Identify Controller Data
1737   ///
1738   Nvme->ControllerData->Sn[19] = 0;
1739   Nvme->ControllerData->Mn[39] = 0;
1740   //NvmeDumpIdentifyController (Nvme->ControllerData);
1741
1742   ///
1743   /// Get current Identify Namespace Data
1744   ///
1745   Nvme->NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)NVME_NAMESPACE_DATA_BASE (Nvme);
1746   Status = NvmeIdentifyNamespace (Nvme, Nvme->Nsid, Nvme->NamespaceData);
1747   if (EFI_ERROR(Status)) {
1748     DEBUG ((DEBUG_ERROR, "NvmeIdentifyNamespace fail, Status = %r\n", Status));
1749     goto Done;
1750   }
1751
1752   ///
1753   /// Dump NvmExpress Identify Namespace Data
1754   ///
1755   if (Nvme->NamespaceData->Ncap == 0) {
1756     DEBUG ((DEBUG_ERROR, "Invalid Namespace, Ncap: %lx\n", Nvme->NamespaceData->Ncap));
1757     Status = EFI_DEVICE_ERROR;
1758     goto Done;
1759   }
1760
1761   Nvme->BlockSize = NvmeGetBlockSize (Nvme);
1762   Nvme->LastBlock = NvmeGetLastLba (Nvme);
1763
1764   Nvme->State    = NvmeStatusInit;
1765
1766   return EFI_SUCCESS;
1767
1768 Done:
1769   return Status;
1770 }
1771
1772 /**
1773   Un-initialize the Nvm Express controller.
1774
1775   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1776
1777   @retval EFI_SUCCESS               - The NVM Express Controller is un-initialized successfully.
1778   @retval Others                    - A device error occurred while un-initializing the controller.
1779
1780 **/
1781 EFI_STATUS
1782 NvmeControllerExit (
1783   IN NVME_CONTEXT       *Nvme
1784   )
1785 {
1786   EFI_STATUS            Status;
1787
1788   Status = EFI_SUCCESS;
1789   if (Nvme->State == NvmeStatusInit || Nvme->State == NvmeStatusMax) {
1790     ///
1791     /// Destroy I/O Submission queue.
1792     ///
1793     Status = NvmeDestroyIoSubmissionQueue (Nvme);
1794     if (EFI_ERROR(Status)) {
1795       DEBUG ((DEBUG_ERROR, "NvmeDestroyIoSubmissionQueue fail, Status = %r\n", Status));
1796       return Status;
1797     }
1798
1799     ///
1800     /// Destroy I/O completion queue.
1801     ///
1802     Status = NvmeDestroyIoCompletionQueue (Nvme);
1803     if (EFI_ERROR(Status)) {
1804       DEBUG ((DEBUG_ERROR, "NvmeDestroyIoCompletionQueue fail, Status = %r\n", Status));
1805       return Status;
1806     }
1807
1808     Status = NvmeShutdownController (Nvme);
1809     if (EFI_ERROR(Status)) {
1810       DEBUG ((DEBUG_ERROR, "NvmeShutdownController fail, Status: %r\n", Status));
1811     }
1812   }
1813
1814   ///
1815   /// Disable PCIE decode
1816   ///
1817   PciWrite8  (Nvme->PciBase + NVME_PCIE_PCICMD, 0x0);
1818   PciWrite32 (Nvme->PciBase + 0x10, 0); // MLBAR (BAR0)
1819   PciWrite32 (Nvme->PciBase + 0x14, 0); // MUBAR (BAR1)
1820
1821   Nvme->State = NvmeStatusUnknown;
1822   return Status;
1823 }