SecurityPkg/Tcg/Opal/OpalPasswordSmm/OpalNvmeMode.c

   1 /** @file
   2   Provide functions to initialize NVME controller and perform NVME commands
   3
   4 Copyright (c) 2016 - 2017, Intel Corporation. All rights reserved.<BR>
   5 This program and the accompanying materials
   6 are licensed and made available under the terms and conditions of the BSD License
   7 which accompanies this distribution.  The full text of the license may be found at
   8 http://opensource.org/licenses/bsd-license.php
   9
  10 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12
  13 **/
  14
  15 #include "OpalPasswordSmm.h"
  16
  17
  18 #define ALIGN(v, a)                         (UINTN)((((v) - 1) | ((a) - 1)) + 1)
  19
  20 ///
  21 /// NVME Host controller registers operation
  22 ///
  23 #define NVME_GET_CAP(Nvme, Cap)             NvmeMmioRead  (Cap, Nvme->Nbar + NVME_CAP_OFFSET, sizeof (NVME_CAP))
  24 #define NVME_GET_CC(Nvme, Cc)               NvmeMmioRead  (Cc, Nvme->Nbar + NVME_CC_OFFSET, sizeof (NVME_CC))
  25 #define NVME_SET_CC(Nvme, Cc)               NvmeMmioWrite (Nvme->Nbar + NVME_CC_OFFSET, Cc, sizeof (NVME_CC))
  26 #define NVME_GET_CSTS(Nvme, Csts)           NvmeMmioRead  (Csts, Nvme->Nbar + NVME_CSTS_OFFSET, sizeof (NVME_CSTS))
  27 #define NVME_GET_AQA(Nvme, Aqa)             NvmeMmioRead  (Aqa, Nvme->Nbar + NVME_AQA_OFFSET, sizeof (NVME_AQA))
  28 #define NVME_SET_AQA(Nvme, Aqa)             NvmeMmioWrite (Nvme->Nbar + NVME_AQA_OFFSET, Aqa, sizeof (NVME_AQA))
  29 #define NVME_GET_ASQ(Nvme, Asq)             NvmeMmioRead  (Asq, Nvme->Nbar + NVME_ASQ_OFFSET, sizeof (NVME_ASQ))
  30 #define NVME_SET_ASQ(Nvme, Asq)             NvmeMmioWrite (Nvme->Nbar + NVME_ASQ_OFFSET, Asq, sizeof (NVME_ASQ))
  31 #define NVME_GET_ACQ(Nvme, Acq)             NvmeMmioRead  (Acq, Nvme->Nbar + NVME_ACQ_OFFSET, sizeof (NVME_ACQ))
  32 #define NVME_SET_ACQ(Nvme, Acq)             NvmeMmioWrite (Nvme->Nbar + NVME_ACQ_OFFSET, Acq, sizeof (NVME_ACQ))
  33 #define NVME_GET_VER(Nvme, Ver)             NvmeMmioRead  (Ver, Nvme->Nbar + NVME_VER_OFFSET, sizeof (NVME_VER))
  34 #define NVME_SET_SQTDBL(Nvme, Qid, Sqtdbl)  NvmeMmioWrite (Nvme->Nbar + NVME_SQTDBL_OFFSET(Qid, Nvme->Cap.Dstrd), Sqtdbl, sizeof (NVME_SQTDBL))
  35 #define NVME_SET_CQHDBL(Nvme, Qid, Cqhdbl)  NvmeMmioWrite (Nvme->Nbar + NVME_CQHDBL_OFFSET(Qid, Nvme->Cap.Dstrd), Cqhdbl, sizeof (NVME_CQHDBL))
  36
  37 ///
  38 /// Base memory address
  39 ///
  40 enum {
  41   BASEMEM_CONTROLLER_DATA,
  42   BASEMEM_IDENTIFY_DATA,
  43   BASEMEM_ASQ,
  44   BASEMEM_ACQ,
  45   BASEMEM_SQ,
  46   BASEMEM_CQ,
  47   BASEMEM_PRP,
  48   BASEMEM_SECURITY,
  49   MAX_BASEMEM_COUNT
  50 };
  51
  52 ///
  53 /// All of base memories are 4K(0x1000) alignment
  54 ///
  55 #define NVME_MEM_BASE(Nvme)                 ((UINTN)(Nvme->BaseMem))
  56 #define NVME_CONTROL_DATA_BASE(Nvme)        (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_CONTROLLER_DATA))                        * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  57 #define NVME_NAMESPACE_DATA_BASE(Nvme)      (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_IDENTIFY_DATA))                          * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  58 #define NVME_ASQ_BASE(Nvme)                 (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_ASQ))                                    * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  59 #define NVME_ACQ_BASE(Nvme)                 (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_ACQ))                                    * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  60 #define NVME_SQ_BASE(Nvme, index)           (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_SQ) + ((index)*(NVME_MAX_IO_QUEUES-1)))  * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  61 #define NVME_CQ_BASE(Nvme, index)           (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_CQ) + ((index)*(NVME_MAX_IO_QUEUES-1)))  * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  62 #define NVME_PRP_BASE(Nvme, index)          (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_PRP) + ((index)*NVME_PRP_SIZE))          * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  63 #define NVME_SEC_BASE(Nvme)                 (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_SECURITY))                               * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  64
  65 /**
  66   Transfer MMIO Data to memory.
  67
  68   @param[in,out] MemBuffer - Destination: Memory address
  69   @param[in] MmioAddr      - Source: MMIO address
  70   @param[in] Size          - Size for read
  71
  72   @retval EFI_SUCCESS - MMIO read sucessfully
  73 **/
  74 EFI_STATUS
  75 NvmeMmioRead (
  76   IN OUT VOID *MemBuffer,
  77   IN     UINTN MmioAddr,
  78   IN     UINTN Size
  79   )
  80 {
  81   UINTN  Offset;
  82   UINT8  Data;
  83   UINT8  *Ptr;
  84
  85   // priority has adjusted
  86   switch (Size) {
  87     case 4:
  88       *((UINT32 *)MemBuffer) = MmioRead32 (MmioAddr);
  89       break;
  90
  91     case 8:
  92       *((UINT64 *)MemBuffer) = MmioRead64 (MmioAddr);
  93       break;
  94
  95     case 2:
  96       *((UINT16 *)MemBuffer) = MmioRead16 (MmioAddr);
  97       break;
  98
  99     case 1:
 100       *((UINT8 *)MemBuffer) = MmioRead8 (MmioAddr);
 101       break;
 102
 103     default:
 104       Ptr = (UINT8 *)MemBuffer;
 105       for (Offset = 0; Offset < Size; Offset += 1) {
 106         Data = MmioRead8 (MmioAddr + Offset);
 107         Ptr[Offset] = Data;
 108       }
 109       break;
 110   }
 111
 112   return EFI_SUCCESS;
 113 }
 114
 115 /**
 116   Transfer memory data to MMIO.
 117
 118   @param[in,out] MmioAddr - Destination: MMIO address
 119   @param[in] MemBuffer    - Source: Memory address
 120   @param[in] Size         - Size for write
 121
 122   @retval EFI_SUCCESS - MMIO write sucessfully
 123 **/
 124 EFI_STATUS
 125 NvmeMmioWrite (
 126   IN OUT UINTN MmioAddr,
 127   IN     VOID *MemBuffer,
 128   IN     UINTN Size
 129   )
 130 {
 131   UINTN  Offset;
 132   UINT8  Data;
 133   UINT8  *Ptr;
 134
 135   // priority has adjusted
 136   switch (Size) {
 137     case 4:
 138       MmioWrite32 (MmioAddr, *((UINT32 *)MemBuffer));
 139       break;
 140
 141     case 8:
 142       MmioWrite64 (MmioAddr, *((UINT64 *)MemBuffer));
 143       break;
 144
 145     case 2:
 146       MmioWrite16 (MmioAddr, *((UINT16 *)MemBuffer));
 147       break;
 148
 149     case 1:
 150       MmioWrite8 (MmioAddr, *((UINT8 *)MemBuffer));
 151       break;
 152
 153     default:
 154       Ptr = (UINT8 *)MemBuffer;
 155       for (Offset = 0; Offset < Size; Offset += 1) {
 156         Data = Ptr[Offset];
 157         MmioWrite8 (MmioAddr + Offset, Data);
 158       }
 159       break;
 160   }
 161
 162   return EFI_SUCCESS;
 163 }
 164
 165 /**
 166   Transfer MMIO data to memory.
 167
 168   @param[in,out] MemBuffer - Destination: Memory address
 169   @param[in] MmioAddr      - Source: MMIO address
 170   @param[in] Size          - Size for read
 171
 172   @retval EFI_SUCCESS - MMIO read sucessfully
 173 **/
 174 EFI_STATUS
 175 OpalPciRead (
 176   IN OUT VOID *MemBuffer,
 177   IN     UINTN MmioAddr,
 178   IN     UINTN Size
 179   )
 180 {
 181   UINTN  Offset;
 182   UINT8  Data;
 183   UINT8  *Ptr;
 184
 185   // priority has adjusted
 186   switch (Size) {
 187     case 4:
 188       *((UINT32 *)MemBuffer) = PciRead32 (MmioAddr);
 189       break;
 190
 191     case 2:
 192       *((UINT16 *)MemBuffer) = PciRead16 (MmioAddr);
 193       break;
 194
 195     case 1:
 196       *((UINT8 *)MemBuffer) = PciRead8 (MmioAddr);
 197       break;
 198
 199     default:
 200       Ptr = (UINT8 *)MemBuffer;
 201       for (Offset = 0; Offset < Size; Offset += 1) {
 202         Data = PciRead8 (MmioAddr + Offset);
 203         Ptr[Offset] = Data;
 204       }
 205       break;
 206   }
 207
 208   return EFI_SUCCESS;
 209 }
 210
 211 /**
 212   Transfer memory data to MMIO.
 213
 214   @param[in,out] MmioAddr - Destination: MMIO address
 215   @param[in] MemBuffer    - Source: Memory address
 216   @param[in] Size         - Size for write
 217
 218   @retval EFI_SUCCESS - MMIO write sucessfully
 219 **/
 220 EFI_STATUS
 221 OpalPciWrite (
 222   IN OUT UINTN MmioAddr,
 223   IN     VOID *MemBuffer,
 224   IN     UINTN Size
 225   )
 226 {
 227   UINTN  Offset;
 228   UINT8  Data;
 229   UINT8  *Ptr;
 230
 231   // priority has adjusted
 232   switch (Size) {
 233     case 4:
 234       PciWrite32 (MmioAddr, *((UINT32 *)MemBuffer));
 235       break;
 236
 237     case 2:
 238       PciWrite16 (MmioAddr, *((UINT16 *)MemBuffer));
 239       break;
 240
 241     case 1:
 242       PciWrite8 (MmioAddr, *((UINT8 *)MemBuffer));
 243       break;
 244
 245     default:
 246       Ptr = (UINT8 *)MemBuffer;
 247       for (Offset = 0; Offset < Size; Offset += 1) {
 248         Data = Ptr[Offset];
 249         PciWrite8 (MmioAddr + Offset, Data);
 250       }
 251       break;
 252   }
 253
 254   return EFI_SUCCESS;
 255 }
 256
 257 /**
 258   Get total pages for specific NVME based memory.
 259
 260   @param[in] BaseMemIndex           - The Index of BaseMem (0-based).
 261
 262   @retval - The page count for specific BaseMem Index
 263
 264 **/
 265 UINT32
 266 NvmeGetBaseMemPages (
 267   IN UINTN              BaseMemIndex
 268   )
 269 {
 270   UINT32                Pages;
 271   UINTN                 Index;
 272   UINT32                PageSizeList[8];
 273
 274   PageSizeList[0] = 1;  /* Controller Data */
 275   PageSizeList[1] = 1;  /* Identify Data */
 276   PageSizeList[2] = 1;  /* ASQ */
 277   PageSizeList[3] = 1;  /* ACQ */
 278   PageSizeList[4] = 1;  /* SQs */
 279   PageSizeList[5] = 1;  /* CQs */
 280   PageSizeList[6] = NVME_PRP_SIZE * NVME_CSQ_DEPTH;  /* PRPs */
 281   PageSizeList[7] = 1;  /* Security Commands */
 282
 283   if (BaseMemIndex > MAX_BASEMEM_COUNT) {
 284     ASSERT (FALSE);
 285     return 0;
 286   }
 287
 288   Pages = 0;
 289   for (Index = 0; Index < BaseMemIndex; Index++) {
 290     Pages += PageSizeList[Index];
 291   }
 292
 293   return Pages;
 294 }
 295
 296 /**
 297   Wait for NVME controller status to be ready or not.
 298
 299   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 300   @param[in] WaitReady              - Flag for waitting status ready or not
 301
 302   @return EFI_SUCCESS               - Successfully to wait specific status.
 303   @return others                    - Fail to wait for specific controller status.
 304
 305 **/
 306 STATIC
 307 EFI_STATUS
 308 NvmeWaitController (
 309   IN NVME_CONTEXT       *Nvme,
 310   IN BOOLEAN            WaitReady
 311   )
 312 {
 313   NVME_CSTS              Csts;
 314   EFI_STATUS             Status;
 315   UINT32                 Index;
 316   UINT8                  Timeout;
 317
 318   //
 319   // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
 320   // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
 321   //
 322   if (Nvme->Cap.To == 0) {
 323     Timeout = 1;
 324   } else {
 325     Timeout = Nvme->Cap.To;
 326   }
 327
 328   Status = EFI_SUCCESS;
 329   for(Index = (Timeout * 500); Index != 0; --Index) {
 330     MicroSecondDelay (1000);
 331
 332     //
 333     // Check if the controller is initialized
 334     //
 335     Status = NVME_GET_CSTS (Nvme, &Csts);
 336     if (EFI_ERROR(Status)) {
 337       DEBUG ((DEBUG_ERROR, "NVME_GET_CSTS fail, Status = %r\n", Status));
 338       return Status;
 339     }
 340
 341     if ((BOOLEAN) Csts.Rdy == WaitReady) {
 342       break;
 343     }
 344   }
 345
 346   if (Index == 0) {
 347     Status = EFI_TIMEOUT;
 348   }
 349
 350   return Status;
 351 }
 352
 353 /**
 354   Disable the Nvm Express controller.
 355
 356   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 357
 358   @return EFI_SUCCESS               - Successfully disable the controller.
 359   @return others                    - Fail to disable the controller.
 360
 361 **/
 362 STATIC
 363 EFI_STATUS
 364 NvmeDisableController (
 365   IN NVME_CONTEXT       *Nvme
 366   )
 367 {
 368   NVME_CC                Cc;
 369   NVME_CSTS              Csts;
 370   EFI_STATUS             Status;
 371
 372   Status = NVME_GET_CSTS (Nvme, &Csts);
 373
 374   ///
 375   /// Read Controller Configuration Register.
 376   ///
 377   Status = NVME_GET_CC (Nvme, &Cc);
 378   if (EFI_ERROR(Status)) {
 379     DEBUG ((DEBUG_ERROR, "NVME_GET_CC fail, Status = %r\n", Status));
 380     goto Done;
 381   }
 382
 383   if (Cc.En == 1) {
 384     Cc.En = 0;
 385     ///
 386     /// Disable the controller.
 387     ///
 388     Status = NVME_SET_CC (Nvme, &Cc);
 389     if (EFI_ERROR(Status)) {
 390       DEBUG ((DEBUG_ERROR, "NVME_SET_CC fail, Status = %r\n", Status));
 391       goto Done;
 392     }
 393   }
 394
 395   Status = NvmeWaitController (Nvme, FALSE);
 396   if (EFI_ERROR(Status)) {
 397     DEBUG ((DEBUG_ERROR, "NvmeWaitController fail, Status = %r\n", Status));
 398     goto Done;
 399   }
 400
 401   return EFI_SUCCESS;
 402
 403 Done:
 404   DEBUG ((DEBUG_INFO, "NvmeDisableController fail, Status: %r\n", Status));
 405   return Status;
 406 }
 407
 408 /**
 409   Enable the Nvm Express controller.
 410
 411   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 412
 413   @return EFI_SUCCESS               - Successfully enable the controller.
 414   @return EFI_DEVICE_ERROR          - Fail to enable the controller.
 415   @return EFI_TIMEOUT               - Fail to enable the controller in given time slot.
 416
 417 **/
 418 STATIC
 419 EFI_STATUS
 420 NvmeEnableController (
 421   IN NVME_CONTEXT       *Nvme
 422   )
 423 {
 424   NVME_CC                Cc;
 425   EFI_STATUS             Status;
 426
 427   //
 428   // Enable the controller
 429   //
 430   ZeroMem (&Cc, sizeof (NVME_CC));
 431   Cc.En     = 1;
 432   Cc.Iosqes = 6;
 433   Cc.Iocqes = 4;
 434   Status    = NVME_SET_CC (Nvme, &Cc);
 435   if (EFI_ERROR(Status)) {
 436     DEBUG ((DEBUG_ERROR, "NVME_SET_CC fail, Status = %r\n", Status));
 437     goto Done;
 438   }
 439
 440   Status = NvmeWaitController (Nvme, TRUE);
 441   if (EFI_ERROR(Status)) {
 442     DEBUG ((DEBUG_ERROR, "NvmeWaitController fail, Status = %r\n", Status));
 443     goto Done;
 444   }
 445
 446   return EFI_SUCCESS;
 447
 448 Done:
 449   DEBUG ((DEBUG_INFO, "NvmeEnableController fail, Status: %r\n", Status));
 450   return Status;
 451 }
 452
 453 /**
 454   Shutdown the Nvm Express controller.
 455
 456   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 457
 458   @return EFI_SUCCESS               - Successfully shutdown the controller.
 459   @return EFI_DEVICE_ERROR          - Fail to shutdown the controller.
 460   @return EFI_TIMEOUT               - Fail to shutdown the controller in given time slot.
 461
 462 **/
 463 STATIC
 464 EFI_STATUS
 465 NvmeShutdownController (
 466   IN NVME_CONTEXT       *Nvme
 467   )
 468 {
 469   NVME_CC                Cc;
 470   NVME_CSTS              Csts;
 471   EFI_STATUS             Status;
 472   UINT32                 Index;
 473   UINTN                  Timeout;
 474
 475   Status    = NVME_GET_CC (Nvme, &Cc);
 476   if (EFI_ERROR(Status)) {
 477     DEBUG ((DEBUG_ERROR, "NVME_GET_CC fail, Status = %r\n", Status));
 478     return Status;
 479   }
 480
 481   Cc.Shn     = 1; // Normal shutdown
 482
 483   Status    = NVME_SET_CC (Nvme, &Cc);
 484   if (EFI_ERROR(Status)) {
 485     DEBUG ((DEBUG_ERROR, "NVME_SET_CC fail, Status = %r\n", Status));
 486     return Status;
 487   }
 488
 489   Timeout = NVME_GENERIC_TIMEOUT/1000; // ms
 490   for(Index = (UINT32)(Timeout); Index != 0; --Index) {
 491     MicroSecondDelay (1000);
 492
 493     Status = NVME_GET_CSTS (Nvme, &Csts);
 494     if (EFI_ERROR(Status)) {
 495       DEBUG ((DEBUG_ERROR, "NVME_GET_CSTS fail, Status = %r\n", Status));
 496       return Status;
 497     }
 498
 499     if (Csts.Shst == 2) { // Shutdown processing complete
 500       break;
 501     }
 502   }
 503
 504   if (Index == 0) {
 505     Status = EFI_TIMEOUT;
 506   }
 507
 508   return Status;
 509 }
 510
 511 /**
 512   Check the execution status from a given completion queue entry.
 513
 514   @param[in]     Cq                 - A pointer to the NVME_CQ item.
 515
 516 **/
 517 EFI_STATUS
 518 NvmeCheckCqStatus (
 519   IN NVME_CQ             *Cq
 520   )
 521 {
 522   if (Cq->Sct == 0x0 && Cq->Sc == 0x0) {
 523     return EFI_SUCCESS;
 524   }
 525
 526   DEBUG ((DEBUG_INFO, "Dump NVMe Completion Entry Status from [0x%x]:\n", (UINTN)Cq));
 527   DEBUG ((DEBUG_INFO, "  SQ Identifier : [0x%x], Phase Tag : [%d], Cmd Identifier : [0x%x]\n", Cq->Sqid, Cq->Pt, Cq->Cid));
 528   DEBUG ((DEBUG_INFO, "  NVMe Cmd Execution Result - "));
 529
 530   switch (Cq->Sct) {
 531     case 0x0:
 532       switch (Cq->Sc) {
 533         case 0x0:
 534           DEBUG ((DEBUG_INFO, "Successful Completion\n"));
 535           return EFI_SUCCESS;
 536         case 0x1:
 537           DEBUG ((DEBUG_INFO, "Invalid Command Opcode\n"));
 538           break;
 539         case 0x2:
 540           DEBUG ((DEBUG_INFO, "Invalid Field in Command\n"));
 541           break;
 542         case 0x3:
 543           DEBUG ((DEBUG_INFO, "Command ID Conflict\n"));
 544           break;
 545         case 0x4:
 546           DEBUG ((DEBUG_INFO, "Data Transfer Error\n"));
 547           break;
 548         case 0x5:
 549           DEBUG ((DEBUG_INFO, "Commands Aborted due to Power Loss Notification\n"));
 550           break;
 551         case 0x6:
 552           DEBUG ((DEBUG_INFO, "Internal Device Error\n"));
 553           break;
 554         case 0x7:
 555           DEBUG ((DEBUG_INFO, "Command Abort Requested\n"));
 556           break;
 557         case 0x8:
 558           DEBUG ((DEBUG_INFO, "Command Aborted due to SQ Deletion\n"));
 559           break;
 560         case 0x9:
 561           DEBUG ((DEBUG_INFO, "Command Aborted due to Failed Fused Command\n"));
 562           break;
 563         case 0xA:
 564           DEBUG ((DEBUG_INFO, "Command Aborted due to Missing Fused Command\n"));
 565           break;
 566         case 0xB:
 567           DEBUG ((DEBUG_INFO, "Invalid Namespace or Format\n"));
 568           break;
 569         case 0xC:
 570           DEBUG ((DEBUG_INFO, "Command Sequence Error\n"));
 571           break;
 572         case 0xD:
 573           DEBUG ((DEBUG_INFO, "Invalid SGL Last Segment Descriptor\n"));
 574           break;
 575         case 0xE:
 576           DEBUG ((DEBUG_INFO, "Invalid Number of SGL Descriptors\n"));
 577           break;
 578         case 0xF:
 579           DEBUG ((DEBUG_INFO, "Data SGL Length Invalid\n"));
 580           break;
 581         case 0x10:
 582           DEBUG ((DEBUG_INFO, "Metadata SGL Length Invalid\n"));
 583           break;
 584         case 0x11:
 585           DEBUG ((DEBUG_INFO, "SGL Descriptor Type Invalid\n"));
 586           break;
 587         case 0x80:
 588           DEBUG ((DEBUG_INFO, "LBA Out of Range\n"));
 589           break;
 590         case 0x81:
 591           DEBUG ((DEBUG_INFO, "Capacity Exceeded\n"));
 592           break;
 593         case 0x82:
 594           DEBUG ((DEBUG_INFO, "Namespace Not Ready\n"));
 595           break;
 596         case 0x83:
 597           DEBUG ((DEBUG_INFO, "Reservation Conflict\n"));
 598           break;
 599       }
 600       break;
 601
 602     case 0x1:
 603       switch (Cq->Sc) {
 604         case 0x0:
 605           DEBUG ((DEBUG_INFO, "Completion Queue Invalid\n"));
 606           break;
 607         case 0x1:
 608           DEBUG ((DEBUG_INFO, "Invalid Queue Identifier\n"));
 609           break;
 610         case 0x2:
 611           DEBUG ((DEBUG_INFO, "Maximum Queue Size Exceeded\n"));
 612           break;
 613         case 0x3:
 614           DEBUG ((DEBUG_INFO, "Abort Command Limit Exceeded\n"));
 615           break;
 616         case 0x5:
 617           DEBUG ((DEBUG_INFO, "Asynchronous Event Request Limit Exceeded\n"));
 618           break;
 619         case 0x6:
 620           DEBUG ((DEBUG_INFO, "Invalid Firmware Slot\n"));
 621           break;
 622         case 0x7:
 623           DEBUG ((DEBUG_INFO, "Invalid Firmware Image\n"));
 624           break;
 625         case 0x8:
 626           DEBUG ((DEBUG_INFO, "Invalid Interrupt Vector\n"));
 627           break;
 628         case 0x9:
 629           DEBUG ((DEBUG_INFO, "Invalid Log Page\n"));
 630           break;
 631         case 0xA:
 632           DEBUG ((DEBUG_INFO, "Invalid Format\n"));
 633           break;
 634         case 0xB:
 635           DEBUG ((DEBUG_INFO, "Firmware Application Requires Conventional Reset\n"));
 636           break;
 637         case 0xC:
 638           DEBUG ((DEBUG_INFO, "Invalid Queue Deletion\n"));
 639           break;
 640         case 0xD:
 641           DEBUG ((DEBUG_INFO, "Feature Identifier Not Saveable\n"));
 642           break;
 643         case 0xE:
 644           DEBUG ((DEBUG_INFO, "Feature Not Changeable\n"));
 645           break;
 646         case 0xF:
 647           DEBUG ((DEBUG_INFO, "Feature Not Namespace Specific\n"));
 648           break;
 649         case 0x10:
 650           DEBUG ((DEBUG_INFO, "Firmware Application Requires NVM Subsystem Reset\n"));
 651           break;
 652         case 0x80:
 653           DEBUG ((DEBUG_INFO, "Conflicting Attributes\n"));
 654           break;
 655         case 0x81:
 656           DEBUG ((DEBUG_INFO, "Invalid Protection Information\n"));
 657           break;
 658         case 0x82:
 659           DEBUG ((DEBUG_INFO, "Attempted Write to Read Only Range\n"));
 660           break;
 661       }
 662       break;
 663
 664     case 0x2:
 665       switch (Cq->Sc) {
 666         case 0x80:
 667           DEBUG ((DEBUG_INFO, "Write Fault\n"));
 668           break;
 669         case 0x81:
 670           DEBUG ((DEBUG_INFO, "Unrecovered Read Error\n"));
 671           break;
 672         case 0x82:
 673           DEBUG ((DEBUG_INFO, "End-to-end Guard Check Error\n"));
 674           break;
 675         case 0x83:
 676           DEBUG ((DEBUG_INFO, "End-to-end Application Tag Check Error\n"));
 677           break;
 678         case 0x84:
 679           DEBUG ((DEBUG_INFO, "End-to-end Reference Tag Check Error\n"));
 680           break;
 681         case 0x85:
 682           DEBUG ((DEBUG_INFO, "Compare Failure\n"));
 683           break;
 684         case 0x86:
 685           DEBUG ((DEBUG_INFO, "Access Denied\n"));
 686           break;
 687       }
 688       break;
 689
 690     default:
 691       DEBUG ((DEBUG_INFO, "Unknown error\n"));
 692       break;
 693   }
 694
 695   return EFI_DEVICE_ERROR;
 696 }
 697
 698 /**
 699   Create PRP lists for Data transfer which is larger than 2 memory pages.
 700   Note here we calcuate the number of required PRP lists and allocate them at one time.
 701
 702   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 703   @param[in] SqId                   - The SQ index for this PRP
 704   @param[in] PhysicalAddr           - The physical base address of Data Buffer.
 705   @param[in] Pages                  - The number of pages to be transfered.
 706   @param[out] PrpListHost           - The host base address of PRP lists.
 707   @param[in,out] PrpListNo          - The number of PRP List.
 708
 709   @retval The pointer Value to the first PRP List of the PRP lists.
 710
 711 **/
 712 STATIC
 713 UINT64
 714 NvmeCreatePrpList (
 715   IN     NVME_CONTEXT                 *Nvme,
 716   IN     UINT16                       SqId,
 717   IN     EFI_PHYSICAL_ADDRESS         PhysicalAddr,
 718   IN     UINTN                        Pages,
 719      OUT VOID                         **PrpListHost,
 720   IN OUT UINTN                        *PrpListNo
 721   )
 722 {
 723   UINTN                       PrpEntryNo;
 724   UINT64                      PrpListBase;
 725   UINTN                       PrpListIndex;
 726   UINTN                       PrpEntryIndex;
 727   UINT64                      Remainder;
 728   EFI_PHYSICAL_ADDRESS        PrpListPhyAddr;
 729   UINTN                       Bytes;
 730   UINT8                       *PrpEntry;
 731   EFI_PHYSICAL_ADDRESS        NewPhyAddr;
 732
 733   ///
 734   /// The number of Prp Entry in a memory page.
 735   ///
 736   PrpEntryNo = EFI_PAGE_SIZE / sizeof (UINT64);
 737
 738   ///
 739   /// Calculate total PrpList number.
 740   ///
 741   *PrpListNo = (UINTN) DivU64x64Remainder ((UINT64)Pages, (UINT64)PrpEntryNo, &Remainder);
 742   if (Remainder != 0) {
 743     *PrpListNo += 1;
 744   }
 745
 746   if (*PrpListNo > NVME_PRP_SIZE) {
 747     DEBUG ((DEBUG_INFO, "NvmeCreatePrpList (PhysicalAddr: %lx, Pages: %x) PrpEntryNo: %x\n",
 748       PhysicalAddr, Pages, PrpEntryNo));
 749     DEBUG ((DEBUG_INFO, "*PrpListNo: %x, Remainder: %lx", *PrpListNo, Remainder));
 750     ASSERT (FALSE);
 751   }
 752   *PrpListHost = (VOID *)(UINTN) NVME_PRP_BASE (Nvme, SqId);
 753
 754   Bytes = EFI_PAGES_TO_SIZE (*PrpListNo);
 755   PrpListPhyAddr = (UINT64)(UINTN)(*PrpListHost);
 756
 757   ///
 758   /// Fill all PRP lists except of last one.
 759   ///
 760   ZeroMem (*PrpListHost, Bytes);
 761   for (PrpListIndex = 0; PrpListIndex < *PrpListNo - 1; ++PrpListIndex) {
 762     PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 763
 764     for (PrpEntryIndex = 0; PrpEntryIndex < PrpEntryNo; ++PrpEntryIndex) {
 765       PrpEntry = (UINT8 *)(UINTN) (PrpListBase + PrpEntryIndex * sizeof(UINT64));
 766       if (PrpEntryIndex != PrpEntryNo - 1) {
 767         ///
 768         /// Fill all PRP entries except of last one.
 769         ///
 770         CopyMem (PrpEntry, (VOID *)(UINTN) (&PhysicalAddr), sizeof (UINT64));
 771         PhysicalAddr += EFI_PAGE_SIZE;
 772       } else {
 773         ///
 774         /// Fill last PRP entries with next PRP List pointer.
 775         ///
 776         NewPhyAddr = (PrpListPhyAddr + (PrpListIndex + 1) * EFI_PAGE_SIZE);
 777         CopyMem (PrpEntry, (VOID *)(UINTN) (&NewPhyAddr), sizeof (UINT64));
 778       }
 779     }
 780   }
 781
 782   ///
 783   /// Fill last PRP list.
 784   ///
 785   PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 786   for (PrpEntryIndex = 0; PrpEntryIndex < ((Remainder != 0) ? Remainder : PrpEntryNo); ++PrpEntryIndex) {
 787     PrpEntry = (UINT8 *)(UINTN) (PrpListBase + PrpEntryIndex * sizeof(UINT64));
 788     CopyMem (PrpEntry, (VOID *)(UINTN) (&PhysicalAddr), sizeof (UINT64));
 789
 790     PhysicalAddr += EFI_PAGE_SIZE;
 791   }
 792
 793   return PrpListPhyAddr;
 794 }
 795
 796 /**
 797   Check whether there are available command slots.
 798
 799   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 800   @param[in] Qid                    - Queue index
 801
 802   @retval EFI_SUCCESS               - Available command slot is found
 803   @retval EFI_NOT_READY             - No available command slot is found
 804   @retval EFI_DEVICE_ERROR          - Error occurred on device side.
 805
 806 **/
 807 EFI_STATUS
 808 NvmeHasFreeCmdSlot (
 809   IN NVME_CONTEXT       *Nvme,
 810   IN UINT8              Qid
 811   )
 812 {
 813   return TRUE;
 814 }
 815
 816 /**
 817   Check whether all command slots are clean.
 818
 819   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 820   @param[in] Qid                    - Queue index
 821
 822   @retval EFI_SUCCESS               - All command slots are clean
 823   @retval EFI_NOT_READY             - Not all command slots are clean
 824   @retval EFI_DEVICE_ERROR          - Error occurred on device side.
 825
 826 **/
 827 EFI_STATUS
 828 NvmeIsAllCmdSlotClean (
 829   IN NVME_CONTEXT       *Nvme,
 830   IN UINT8              Qid
 831   )
 832 {
 833   return EFI_SUCCESS;
 834 }
 835
 836 /**
 837   Waits until all NVME commands completed.
 838
 839   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 840   @param[in] Qid                    - Queue index
 841
 842   @retval EFI_SUCCESS               - All NVME commands have completed
 843   @retval EFI_TIMEOUT               - Timeout occured
 844   @retval EFI_NOT_READY             - Not all NVME commands have completed
 845   @retval others                    - Error occurred on device side.
 846 **/
 847 EFI_STATUS
 848 NvmeWaitAllComplete (
 849   IN NVME_CONTEXT       *Nvme,
 850   IN UINT8              Qid
 851   )
 852 {
 853   return EFI_SUCCESS;
 854 }
 855
 856 /**
 857   Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function supports
 858   both blocking I/O and nonblocking I/O. The blocking I/O functionality is required, and the nonblocking
 859   I/O functionality is optional.
 860
 861   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 862   @param[in] NamespaceId            - Is a 32 bit Namespace ID to which the Express HCI command packet will be sent.
 863                                       A Value of 0 denotes the NVM Express controller, a Value of all 0FFh in the namespace
 864                                       ID specifies that the command packet should be sent to all valid namespaces.
 865   @param[in] NamespaceUuid          - Is a 64 bit Namespace UUID to which the Express HCI command packet will be sent.
 866                                       A Value of 0 denotes the NVM Express controller, a Value of all 0FFh in the namespace
 867                                       UUID specifies that the command packet should be sent to all valid namespaces.
 868   @param[in,out] Packet             - A pointer to the NVM Express HCI Command Packet to send to the NVMe namespace specified
 869                                       by NamespaceId.
 870
 871   @retval EFI_SUCCESS               - The NVM Express Command Packet was sent by the host. TransferLength bytes were transferred
 872                                       to, or from DataBuffer.
 873   @retval EFI_NOT_READY             - The NVM Express Command Packet could not be sent because the controller is not ready. The caller
 874                                       may retry again later.
 875   @retval EFI_DEVICE_ERROR          - A device error occurred while attempting to send the NVM Express Command Packet.
 876   @retval EFI_INVALID_PARAMETER     - Namespace, or the contents of NVM_EXPRESS_PASS_THRU_COMMAND_PACKET are invalid. The NVM
 877                                       Express Command Packet was not sent, so no additional status information is available.
 878   @retval EFI_UNSUPPORTED           - The command described by the NVM Express Command Packet is not supported by the host adapter.
 879                                       The NVM Express Command Packet was not sent, so no additional status information is available.
 880   @retval EFI_TIMEOUT               - A timeout occurred while waiting for the NVM Express Command Packet to execute.
 881
 882 **/
 883 EFI_STATUS
 884 NvmePassThru (
 885   IN     NVME_CONTEXT                         *Nvme,
 886   IN     UINT32                               NamespaceId,
 887   IN     UINT64                               NamespaceUuid,
 888   IN OUT NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *Packet
 889   )
 890 {
 891   EFI_STATUS                    Status;
 892   NVME_SQ                       *Sq;
 893   NVME_CQ                       *Cq;
 894   UINT8                         Qid;
 895   UINT32                        Bytes;
 896   UINT32                        Offset;
 897   EFI_PHYSICAL_ADDRESS          PhyAddr;
 898   VOID                          *PrpListHost;
 899   UINTN                         PrpListNo;
 900   UINT32                        Timer;
 901   UINTN SqSize;
 902   UINTN CqSize;
 903
 904   ///
 905   /// check the Data fields in Packet parameter.
 906   ///
 907   if ((Nvme == NULL) || (Packet == NULL)) {
 908     DEBUG ((DEBUG_ERROR, "NvmePassThru, invalid parameter: Nvme(%x)/Packet(%x)\n",
 909       (UINTN)Nvme, (UINTN)Packet));
 910     return EFI_INVALID_PARAMETER;
 911   }
 912
 913   if ((Packet->NvmeCmd == NULL) || (Packet->NvmeResponse == NULL)) {
 914     DEBUG ((DEBUG_ERROR, "NvmePassThru, invalid parameter: NvmeCmd(%x)/NvmeResponse(%x)\n",
 915       (UINTN)Packet->NvmeCmd, (UINTN)Packet->NvmeResponse));
 916     return EFI_INVALID_PARAMETER;
 917   }
 918
 919   if (Packet->QueueId != NVME_ADMIN_QUEUE && Packet->QueueId != NVME_IO_QUEUE) {
 920     DEBUG ((DEBUG_ERROR, "NvmePassThru, invalid parameter: QueueId(%x)\n",
 921       Packet->QueueId));
 922     return EFI_INVALID_PARAMETER;
 923   }
 924
 925   PrpListHost = NULL;
 926   PrpListNo   = 0;
 927   Status      = EFI_SUCCESS;
 928
 929   Qid = Packet->QueueId;
 930   Sq  = Nvme->SqBuffer[Qid] + Nvme->SqTdbl[Qid].Sqt;
 931   Cq  = Nvme->CqBuffer[Qid] + Nvme->CqHdbl[Qid].Cqh;
 932   if (Qid == NVME_ADMIN_QUEUE) {
 933     SqSize = NVME_ASQ_SIZE + 1;
 934     CqSize = NVME_ACQ_SIZE + 1;
 935   } else {
 936     SqSize = NVME_CSQ_DEPTH;
 937     CqSize = NVME_CCQ_DEPTH;
 938   }
 939
 940   if (Packet->NvmeCmd->Nsid != NamespaceId) {
 941     DEBUG ((DEBUG_ERROR, "NvmePassThru: Nsid mismatch (%x, %x)\n",
 942       Packet->NvmeCmd->Nsid, NamespaceId));
 943     return EFI_INVALID_PARAMETER;
 944   }
 945
 946   ZeroMem (Sq, sizeof (NVME_SQ));
 947   Sq->Opc  = Packet->NvmeCmd->Cdw0.Opcode;
 948   Sq->Fuse = Packet->NvmeCmd->Cdw0.FusedOperation;
 949   Sq->Cid  = Packet->NvmeCmd->Cdw0.Cid;
 950   Sq->Nsid = Packet->NvmeCmd->Nsid;
 951
 952   ///
 953   /// Currently we only support PRP for Data transfer, SGL is NOT supported.
 954   ///
 955   ASSERT (Sq->Psdt == 0);
 956   if (Sq->Psdt != 0) {
 957     DEBUG ((DEBUG_ERROR, "NvmePassThru: doesn't support SGL mechanism\n"));
 958     return EFI_UNSUPPORTED;
 959   }
 960
 961   Sq->Prp[0] = Packet->TransferBuffer;
 962   Sq->Prp[1] = 0;
 963
 964   if(Packet->MetadataBuffer != (UINT64)(UINTN)NULL) {
 965     Sq->Mptr = Packet->MetadataBuffer;
 966   }
 967
 968   ///
 969   /// If the Buffer Size spans more than two memory pages (page Size as defined in CC.Mps),
 970   /// then build a PRP list in the second PRP submission queue entry.
 971   ///
 972   Offset = ((UINT32)Sq->Prp[0]) & (EFI_PAGE_SIZE - 1);
 973   Bytes  = Packet->TransferLength;
 974
 975   if ((Offset + Bytes) > (EFI_PAGE_SIZE * 2)) {
 976     ///
 977     /// Create PrpList for remaining Data Buffer.
 978     ///
 979     PhyAddr = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
 980     Sq->Prp[1] = NvmeCreatePrpList (Nvme, Nvme->SqTdbl[Qid].Sqt, PhyAddr, EFI_SIZE_TO_PAGES(Offset + Bytes) - 1, &PrpListHost, &PrpListNo);
 981     if (Sq->Prp[1] == 0) {
 982       Status = EFI_OUT_OF_RESOURCES;
 983       DEBUG ((DEBUG_ERROR, "NvmeCreatePrpList fail, Status: %r\n", Status));
 984       goto EXIT;
 985     }
 986
 987   } else if ((Offset + Bytes) > EFI_PAGE_SIZE) {
 988     Sq->Prp[1] = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
 989   }
 990
 991   if(Packet->NvmeCmd->Flags & CDW10_VALID) {
 992     Sq->Payload.Raw.Cdw10 = Packet->NvmeCmd->Cdw10;
 993   }
 994   if(Packet->NvmeCmd->Flags & CDW11_VALID) {
 995     Sq->Payload.Raw.Cdw11 = Packet->NvmeCmd->Cdw11;
 996   }
 997   if(Packet->NvmeCmd->Flags & CDW12_VALID) {
 998     Sq->Payload.Raw.Cdw12 = Packet->NvmeCmd->Cdw12;
 999   }
1000   if(Packet->NvmeCmd->Flags & CDW13_VALID) {
1001     Sq->Payload.Raw.Cdw13 = Packet->NvmeCmd->Cdw13;
1002   }
1003   if(Packet->NvmeCmd->Flags & CDW14_VALID) {
1004     Sq->Payload.Raw.Cdw14 = Packet->NvmeCmd->Cdw14;
1005   }
1006   if(Packet->NvmeCmd->Flags & CDW15_VALID) {
1007     Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
1008   }
1009
1010   ///
1011   /// Ring the submission queue doorbell.
1012   ///
1013   Nvme->SqTdbl[Qid].Sqt++;
1014   if(Nvme->SqTdbl[Qid].Sqt == SqSize) {
1015     Nvme->SqTdbl[Qid].Sqt = 0;
1016   }
1017   Status = NVME_SET_SQTDBL (Nvme, Qid, &Nvme->SqTdbl[Qid]);
1018   if (EFI_ERROR(Status)) {
1019     DEBUG ((DEBUG_ERROR, "NVME_SET_SQTDBL fail, Status: %r\n", Status));
1020     goto EXIT;
1021   }
1022
1023   ///
1024   /// Wait for completion queue to get filled in.
1025   ///
1026   Status = EFI_TIMEOUT;
1027   Timer   = 0;
1028   while (Timer < NVME_CMD_TIMEOUT) {
1029     //DEBUG ((DEBUG_VERBOSE, "Timer: %x, Cq:\n", Timer));
1030     //DumpMem (Cq, sizeof (NVME_CQ));
1031     if (Cq->Pt != Nvme->Pt[Qid]) {
1032       Status = EFI_SUCCESS;
1033       break;
1034     }
1035
1036     MicroSecondDelay (NVME_CMD_WAIT);
1037     Timer += NVME_CMD_WAIT;
1038   }
1039
1040   Nvme->CqHdbl[Qid].Cqh++;
1041   if (Nvme->CqHdbl[Qid].Cqh == CqSize) {
1042     Nvme->CqHdbl[Qid].Cqh = 0;
1043     Nvme->Pt[Qid] ^= 1;
1044   }
1045
1046   ///
1047   /// Copy the Respose Queue entry for this command to the callers response Buffer
1048   ///
1049   CopyMem (Packet->NvmeResponse, Cq, sizeof(NVM_EXPRESS_RESPONSE));
1050
1051   if (!EFI_ERROR(Status)) { // We still need to check CQ status if no timeout error occured
1052     Status = NvmeCheckCqStatus (Cq);
1053   }
1054   NVME_SET_CQHDBL (Nvme, Qid, &Nvme->CqHdbl[Qid]);
1055
1056 EXIT:
1057   return Status;
1058 }
1059
1060 /**
1061   Get identify controller Data.
1062
1063   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1064   @param[in] Buffer                 - The Buffer used to store the identify controller Data.
1065
1066   @return EFI_SUCCESS               - Successfully get the identify controller Data.
1067   @return others                    - Fail to get the identify controller Data.
1068
1069 **/
1070 STATIC
1071 EFI_STATUS
1072 NvmeIdentifyController (
1073   IN NVME_CONTEXT                          *Nvme,
1074   IN VOID                                  *Buffer
1075   )
1076 {
1077   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1078   NVM_EXPRESS_COMMAND                      Command;
1079   NVM_EXPRESS_RESPONSE                     Response;
1080   EFI_STATUS                               Status;
1081
1082   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1083   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1084   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1085
1086   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_OPC;
1087   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1088   //
1089   // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
1090   // For the Identify command, the Namespace Identifier is only used for the Namespace Data structure.
1091   //
1092   Command.Nsid        = 0;
1093
1094   CommandPacket.NvmeCmd        = &Command;
1095   CommandPacket.NvmeResponse   = &Response;
1096   CommandPacket.TransferBuffer = (UINT64)(UINTN)Buffer;
1097   CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA);
1098   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1099   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1100   //
1101   // Set bit 0 (Cns bit) to 1 to identify a controller
1102   //
1103   Command.Cdw10                = 1;
1104   Command.Flags                = CDW10_VALID;
1105
1106   Status = NvmePassThru (
1107               Nvme,
1108               NVME_CONTROLLER_ID,
1109               0,
1110               &CommandPacket
1111               );
1112   if (!EFI_ERROR (Status)) {
1113     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1114   }
1115
1116   return Status;
1117 }
1118
1119 /**
1120   Get specified identify namespace Data.
1121
1122   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1123   @param[in] NamespaceId            - The specified namespace identifier.
1124   @param[in] Buffer                 - The Buffer used to store the identify namespace Data.
1125
1126   @return EFI_SUCCESS               - Successfully get the identify namespace Data.
1127   @return others                    - Fail to get the identify namespace Data.
1128
1129 **/
1130 STATIC
1131 EFI_STATUS
1132 NvmeIdentifyNamespace (
1133   IN NVME_CONTEXT                          *Nvme,
1134   IN UINT32                                NamespaceId,
1135   IN VOID                                  *Buffer
1136   )
1137 {
1138   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1139   NVM_EXPRESS_COMMAND                      Command;
1140   NVM_EXPRESS_RESPONSE                     Response;
1141   EFI_STATUS                               Status;
1142
1143   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1144   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1145   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1146
1147   CommandPacket.NvmeCmd      = &Command;
1148   CommandPacket.NvmeResponse = &Response;
1149
1150   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_OPC;
1151   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1152   Command.Nsid        = NamespaceId;
1153   CommandPacket.TransferBuffer = (UINT64)(UINTN)Buffer;
1154   CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA);
1155   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1156   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1157   //
1158   // Set bit 0 (Cns bit) to 1 to identify a namespace
1159   //
1160   CommandPacket.NvmeCmd->Cdw10 = 0;
1161   CommandPacket.NvmeCmd->Flags = CDW10_VALID;
1162
1163   Status = NvmePassThru (
1164               Nvme,
1165               NamespaceId,
1166               0,
1167               &CommandPacket
1168               );
1169   if (!EFI_ERROR (Status)) {
1170     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1171   }
1172
1173   return Status;
1174 }
1175
1176 /**
1177   Get Block Size for specific namespace of NVME.
1178
1179   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1180
1181   @return                           - Block Size in bytes
1182
1183 **/
1184 STATIC
1185 UINT32
1186 NvmeGetBlockSize (
1187   IN NVME_CONTEXT       *Nvme
1188   )
1189 {
1190   UINT32                BlockSize;
1191   UINT32                Lbads;
1192   UINT32                Flbas;
1193   UINT32                LbaFmtIdx;
1194
1195   Flbas     = Nvme->NamespaceData->Flbas;
1196   LbaFmtIdx = Flbas & 3;
1197   Lbads     = Nvme->NamespaceData->LbaFormat[LbaFmtIdx].Lbads;
1198
1199   BlockSize = (UINT32)1 << Lbads;
1200   return BlockSize;
1201 }
1202
1203 /**
1204   Get last LBA for specific namespace of NVME.
1205
1206   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1207
1208   @return                           - Last LBA address
1209
1210 **/
1211 STATIC
1212 EFI_LBA
1213 NvmeGetLastLba (
1214   IN NVME_CONTEXT       *Nvme
1215   )
1216 {
1217   EFI_LBA               LastBlock;
1218   LastBlock = Nvme->NamespaceData->Nsze - 1;
1219   return LastBlock;
1220 }
1221
1222 /**
1223   Create io completion queue.
1224
1225   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1226
1227   @return EFI_SUCCESS               - Successfully create io completion queue.
1228   @return others                    - Fail to create io completion queue.
1229
1230 **/
1231 STATIC
1232 EFI_STATUS
1233 NvmeCreateIoCompletionQueue (
1234   IN     NVME_CONTEXT                      *Nvme
1235   )
1236 {
1237   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1238   NVM_EXPRESS_COMMAND                      Command;
1239   NVM_EXPRESS_RESPONSE                     Response;
1240   EFI_STATUS                               Status;
1241   NVME_ADMIN_CRIOCQ                        CrIoCq;
1242
1243   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1244   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1245   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1246   ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ));
1247
1248   CommandPacket.NvmeCmd      = &Command;
1249   CommandPacket.NvmeResponse = &Response;
1250
1251   Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_OPC;
1252   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1253   CommandPacket.TransferBuffer = (UINT64)(UINTN)Nvme->CqBuffer[NVME_IO_QUEUE];
1254   CommandPacket.TransferLength = EFI_PAGE_SIZE;
1255   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1256   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1257
1258   CrIoCq.Qid   = NVME_IO_QUEUE;
1259   CrIoCq.Qsize = NVME_CCQ_SIZE;
1260   CrIoCq.Pc    = 1;
1261   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
1262   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1263
1264   Status = NvmePassThru (
1265               Nvme,
1266               NVME_CONTROLLER_ID,
1267               0,
1268               &CommandPacket
1269               );
1270   if (!EFI_ERROR (Status)) {
1271     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1272   }
1273
1274   return Status;
1275 }
1276
1277 /**
1278   Create io submission queue.
1279
1280   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1281
1282   @return EFI_SUCCESS               - Successfully create io submission queue.
1283   @return others                    - Fail to create io submission queue.
1284
1285 **/
1286 STATIC
1287 EFI_STATUS
1288 NvmeCreateIoSubmissionQueue (
1289   IN NVME_CONTEXT                          *Nvme
1290   )
1291 {
1292   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1293   NVM_EXPRESS_COMMAND                      Command;
1294   NVM_EXPRESS_RESPONSE                     Response;
1295   EFI_STATUS                               Status;
1296   NVME_ADMIN_CRIOSQ                        CrIoSq;
1297
1298   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1299   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1300   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1301   ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ));
1302
1303   CommandPacket.NvmeCmd      = &Command;
1304   CommandPacket.NvmeResponse = &Response;
1305
1306   Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_OPC;
1307   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1308   CommandPacket.TransferBuffer = (UINT64)(UINTN)Nvme->SqBuffer[NVME_IO_QUEUE];
1309   CommandPacket.TransferLength = EFI_PAGE_SIZE;
1310   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1311   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1312
1313   CrIoSq.Qid   = NVME_IO_QUEUE;
1314   CrIoSq.Qsize = NVME_CSQ_SIZE;
1315   CrIoSq.Pc    = 1;
1316   CrIoSq.Cqid  = NVME_IO_QUEUE;
1317   CrIoSq.Qprio = 0;
1318   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
1319   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1320
1321   Status = NvmePassThru (
1322               Nvme,
1323               NVME_CONTROLLER_ID,
1324               0,
1325               &CommandPacket
1326               );
1327   if (!EFI_ERROR (Status)) {
1328     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1329   }
1330
1331   return Status;
1332 }
1333
1334 /**
1335   Security send and receive commands.
1336
1337   @param[in]     Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1338   @param[in]     SendCommand            - The flag to indicate the command type, TRUE for Send command and FALSE for receive command
1339   @param[in]     SecurityProtocol       - Security Protocol
1340   @param[in]     SpSpecific             - Security Protocol Specific
1341   @param[in]     TransferLength         - Transfer Length of Buffer (in bytes) - always a multiple of 512
1342   @param[in,out] TransferBuffer         - Address of Data to transfer
1343
1344   @return EFI_SUCCESS               - Successfully create io submission queue.
1345   @return others                    - Fail to send/receive commands.
1346
1347 **/
1348 EFI_STATUS
1349 NvmeSecuritySendReceive (
1350   IN NVME_CONTEXT                          *Nvme,
1351   IN BOOLEAN                               SendCommand,
1352   IN UINT8                                 SecurityProtocol,
1353   IN UINT16                                SpSpecific,
1354   IN UINTN                                 TransferLength,
1355   IN OUT VOID                              *TransferBuffer
1356   )
1357 {
1358   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1359   NVM_EXPRESS_COMMAND                      Command;
1360   NVM_EXPRESS_RESPONSE                     Response;
1361   EFI_STATUS                               Status;
1362   NVME_ADMIN_SECSEND                       SecSend;
1363   OACS                                     *Oacs;
1364   UINT8                                    Opcode;
1365   VOID*                                    *SecBuff;
1366
1367   Oacs = (OACS *)&Nvme->ControllerData->Oacs;
1368
1369   //
1370   // Verify security bit for Security Send/Receive commands
1371   //
1372   if (Oacs->Security == 0) {
1373     DEBUG ((DEBUG_ERROR, "Security command doesn't support.\n"));
1374     return EFI_NOT_READY;
1375   }
1376
1377   SecBuff = (VOID *)(UINTN) NVME_SEC_BASE (Nvme);
1378
1379   //
1380   // Actions for sending security command
1381   //
1382   if (SendCommand) {
1383     CopyMem (SecBuff, TransferBuffer, TransferLength);
1384   }
1385
1386   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1387   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1388   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1389   ZeroMem (&SecSend, sizeof(NVME_ADMIN_SECSEND));
1390
1391   CommandPacket.NvmeCmd      = &Command;
1392   CommandPacket.NvmeResponse = &Response;
1393
1394   Opcode = (UINT8)(SendCommand ? NVME_ADMIN_SECURITY_SEND_OPC : NVME_ADMIN_SECURITY_RECV_OPC);
1395   Command.Cdw0.Opcode = Opcode;
1396   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1397   CommandPacket.TransferBuffer = (UINT64)(UINTN)SecBuff;
1398   CommandPacket.TransferLength = (UINT32)TransferLength;
1399   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1400   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1401
1402   SecSend.Spsp = SpSpecific;
1403   SecSend.Secp = SecurityProtocol;
1404   SecSend.Tl   = (UINT32)TransferLength;
1405
1406   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &SecSend, sizeof (NVME_ADMIN_SECSEND));
1407   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1408
1409   Status = NvmePassThru (
1410               Nvme,
1411               NVME_CONTROLLER_ID,
1412               0,
1413               &CommandPacket
1414               );
1415   if (!EFI_ERROR (Status)) {
1416     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1417   }
1418
1419   //
1420   // Actions for receiving security command
1421   //
1422   if (!SendCommand) {
1423     CopyMem (TransferBuffer, SecBuff, TransferLength);
1424   }
1425
1426   return Status;
1427 }
1428
1429 /**
1430   Destroy io completion queue.
1431
1432   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1433
1434   @return EFI_SUCCESS               - Successfully destroy io completion queue.
1435   @return others                    - Fail to destroy io completion queue.
1436
1437 **/
1438 STATIC
1439 EFI_STATUS
1440 NvmeDestroyIoCompletionQueue (
1441   IN     NVME_CONTEXT                      *Nvme
1442   )
1443 {
1444   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1445   NVM_EXPRESS_COMMAND                      Command;
1446   NVM_EXPRESS_RESPONSE                     Response;
1447   EFI_STATUS                               Status;
1448   NVME_ADMIN_DEIOCQ                        DelIoCq;
1449
1450   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1451   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1452   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1453   ZeroMem (&DelIoCq, sizeof(NVME_ADMIN_DEIOCQ));
1454
1455   CommandPacket.NvmeCmd      = &Command;
1456   CommandPacket.NvmeResponse = &Response;
1457
1458   Command.Cdw0.Opcode = NVME_ADMIN_DELIOCQ_OPC;
1459   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1460   CommandPacket.TransferBuffer = (UINT64)(UINTN)Nvme->CqBuffer[NVME_IO_QUEUE];
1461   CommandPacket.TransferLength = EFI_PAGE_SIZE;
1462   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1463   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1464
1465   DelIoCq.Qid   = NVME_IO_QUEUE;
1466   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &DelIoCq, sizeof (NVME_ADMIN_DEIOCQ));
1467   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1468
1469   Status = NvmePassThru (
1470               Nvme,
1471               NVME_CONTROLLER_ID,
1472               0,
1473               &CommandPacket
1474               );
1475   if (!EFI_ERROR (Status)) {
1476     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1477   }
1478
1479   return Status;
1480 }
1481
1482 /**
1483   Destroy io submission queue.
1484
1485   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1486
1487   @return EFI_SUCCESS               - Successfully destroy io submission queue.
1488   @return others                    - Fail to destroy io submission queue.
1489
1490 **/
1491 STATIC
1492 EFI_STATUS
1493 NvmeDestroyIoSubmissionQueue (
1494   IN NVME_CONTEXT                          *Nvme
1495   )
1496 {
1497   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1498   NVM_EXPRESS_COMMAND                      Command;
1499   NVM_EXPRESS_RESPONSE                     Response;
1500   EFI_STATUS                               Status;
1501   NVME_ADMIN_DEIOSQ                        DelIoSq;
1502
1503   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1504   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1505   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1506   ZeroMem (&DelIoSq, sizeof(NVME_ADMIN_DEIOSQ));
1507
1508   CommandPacket.NvmeCmd      = &Command;
1509   CommandPacket.NvmeResponse = &Response;
1510
1511   Command.Cdw0.Opcode = NVME_ADMIN_DELIOSQ_OPC;
1512   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1513   CommandPacket.TransferBuffer = (UINT64)(UINTN)Nvme->SqBuffer[NVME_IO_QUEUE];
1514   CommandPacket.TransferLength = EFI_PAGE_SIZE;
1515   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1516   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1517
1518   DelIoSq.Qid   = NVME_IO_QUEUE;
1519   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &DelIoSq, sizeof (NVME_ADMIN_DEIOSQ));
1520   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1521
1522   Status = NvmePassThru (
1523               Nvme,
1524               NVME_CONTROLLER_ID,
1525               0,
1526               &CommandPacket
1527               );
1528   if (!EFI_ERROR (Status)) {
1529     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1530   }
1531
1532   return Status;
1533 }
1534
1535 /**
1536   Allocate transfer-related Data struct which is used at Nvme.
1537
1538   @param[in] ImageHandle         Image handle for this driver image
1539   @param[in] Nvme                The pointer to the NVME_CONTEXT Data structure.
1540
1541   @retval  EFI_OUT_OF_RESOURCE   The allocation is failure.
1542   @retval  EFI_SUCCESS           Successful to allocate memory.
1543
1544 **/
1545 EFI_STATUS
1546 EFIAPI
1547 NvmeAllocateResource (
1548   IN EFI_HANDLE                         ImageHandle,
1549   IN NVME_CONTEXT                       *Nvme
1550   )
1551 {
1552   EFI_STATUS            Status;
1553   EFI_PHYSICAL_ADDRESS  Addr;
1554   UINT32                Size;
1555
1556   //
1557   // Allocate resources required by NVMe host controller.
1558   //
1559   // NBAR
1560   Size = 0x10000;
1561   Addr = 0xFFFFFFFF;
1562   Status = gDS->AllocateMemorySpace (
1563                   EfiGcdAllocateMaxAddressSearchBottomUp,
1564                   EfiGcdMemoryTypeMemoryMappedIo,
1565                   15,                             // 2^15: 32K Alignment
1566                   Size,
1567                   &Addr,
1568                   ImageHandle,
1569                   NULL
1570                   );
1571   if (EFI_ERROR (Status)) {
1572     return EFI_OUT_OF_RESOURCES;
1573   }
1574   Nvme->Nbar = (UINT32) Addr;
1575
1576   // DMA Buffer
1577   Size = NVME_MEM_MAX_SIZE;
1578   Addr = 0xFFFFFFFF;
1579   Status = gBS->AllocatePages (
1580                   AllocateMaxAddress,
1581                   EfiACPIMemoryNVS,
1582                   EFI_SIZE_TO_PAGES (Size),
1583                   (EFI_PHYSICAL_ADDRESS *)&Addr
1584                   );
1585   if (EFI_ERROR (Status)) {
1586     return EFI_OUT_OF_RESOURCES;
1587   }
1588   Nvme->BaseMem = (UINT32) Addr;
1589
1590   // Clean up DMA Buffer before using
1591   ZeroMem ((VOID *)(UINTN)Addr, NVME_MEM_MAX_SIZE);
1592
1593   return EFI_SUCCESS;
1594 }
1595
1596 /**
1597   Free allocated transfer-related Data struct which is used at NVMe.
1598
1599   @param[in] Nvme                The pointer to the NVME_CONTEXT Data structure.
1600
1601 **/
1602 VOID
1603 EFIAPI
1604 NvmeFreeResource (
1605   IN NVME_CONTEXT                       *Nvme
1606   )
1607 {
1608   UINT32                Size;
1609
1610   // NBAR
1611   if (Nvme->BaseMem != 0) {
1612     Size = 0x10000;
1613     gDS->FreeMemorySpace (Nvme->Nbar, Size);
1614   }
1615
1616   // DMA Buffer
1617   if (Nvme->Nbar != 0) {
1618     Size = NVME_MEM_MAX_SIZE;
1619     gBS->FreePages ((EFI_PHYSICAL_ADDRESS)(UINTN) Nvme->Nbar, EFI_SIZE_TO_PAGES (Size));
1620   }
1621 }
1622
1623
1624 /**
1625   Initialize the Nvm Express controller.
1626
1627   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1628
1629   @retval EFI_SUCCESS               - The NVM Express Controller is initialized successfully.
1630   @retval Others                    - A device error occurred while initializing the controller.
1631
1632 **/
1633 EFI_STATUS
1634 NvmeControllerInit (
1635   IN NVME_CONTEXT       *Nvme
1636   )
1637 {
1638   EFI_STATUS            Status;
1639   NVME_AQA              Aqa;
1640   NVME_ASQ              Asq;
1641   NVME_ACQ              Acq;
1642   NVME_VER              Ver;
1643
1644   UINT32                MlBAR;
1645   UINT32                MuBAR;
1646
1647   ///
1648   /// Update PCIE BAR0/1 for NVME device
1649   ///
1650   MlBAR = Nvme->Nbar;
1651   MuBAR = 0;
1652   PciWrite32 (Nvme->PciBase + 0x10, MlBAR); // MLBAR (BAR0)
1653   PciWrite32 (Nvme->PciBase + 0x14, MuBAR); // MUBAR (BAR1)
1654
1655   ///
1656   /// Enable PCIE decode
1657   ///
1658   PciWrite8 (Nvme->PciBase + NVME_PCIE_PCICMD, 0x6);
1659
1660   // Version
1661   NVME_GET_VER (Nvme, &Ver);
1662   if (!(Ver.Mjr == 0x0001) && (Ver.Mnr == 0x0000)) {
1663     DEBUG ((DEBUG_INFO, "\n!!!\n!!! NVME Version mismatch for the implementation !!!\n!!!\n"));
1664   }
1665
1666   ///
1667   /// Read the Controller Capabilities register and verify that the NVM command set is supported
1668   ///
1669   Status = NVME_GET_CAP (Nvme, &Nvme->Cap);
1670   if (EFI_ERROR (Status)) {
1671     DEBUG ((DEBUG_ERROR, "NVME_GET_CAP fail, Status: %r\n", Status));
1672     goto Done;
1673   }
1674
1675   if (Nvme->Cap.Css != 0x01) {
1676     DEBUG ((DEBUG_ERROR, "NvmeControllerInit fail: the controller doesn't support NVMe command set\n"));
1677     Status = EFI_UNSUPPORTED;
1678     goto Done;
1679   }
1680
1681   ///
1682   /// Currently the driver only supports 4k page Size.
1683   ///
1684   if ((Nvme->Cap.Mpsmin + 12) > EFI_PAGE_SHIFT) {
1685     DEBUG ((DEBUG_ERROR, "NvmeControllerInit fail: only supports 4k page Size\n"));
1686     ASSERT (FALSE);
1687     Status = EFI_UNSUPPORTED;
1688     goto Done;
1689   }
1690
1691   Nvme->Cid[0] = 0;
1692   Nvme->Cid[1] = 0;
1693
1694   Nvme->Pt[0]  = 0;
1695   Nvme->Pt[1]  = 0;
1696
1697   ZeroMem ((VOID *)(UINTN)(&(Nvme->SqTdbl[0])), sizeof (NVME_SQTDBL) * NVME_MAX_IO_QUEUES);
1698   ZeroMem ((VOID *)(UINTN)(&(Nvme->CqHdbl[0])), sizeof (NVME_CQHDBL) * NVME_MAX_IO_QUEUES);
1699
1700   ZeroMem ((VOID *)(UINTN)Nvme->BaseMem, NVME_MEM_MAX_SIZE);
1701
1702   Status = NvmeDisableController (Nvme);
1703   if (EFI_ERROR(Status)) {
1704     DEBUG ((DEBUG_ERROR, "NvmeDisableController fail, Status: %r\n", Status));
1705     goto Done;
1706   }
1707
1708   ///
1709   /// set number of entries admin submission & completion queues.
1710   ///
1711   Aqa.Asqs  = NVME_ASQ_SIZE;
1712   Aqa.Rsvd1 = 0;
1713   Aqa.Acqs  = NVME_ACQ_SIZE;
1714   Aqa.Rsvd2 = 0;
1715
1716   ///
1717   /// Address of admin submission queue.
1718   ///
1719   Asq = (UINT64)(UINTN)(NVME_ASQ_BASE (Nvme) & ~0xFFF);
1720
1721   ///
1722   /// Address of admin completion queue.
1723   ///
1724   Acq = (UINT64)(UINTN)(NVME_ACQ_BASE (Nvme) & ~0xFFF);
1725
1726   ///
1727   /// Address of I/O submission & completion queue.
1728   ///
1729   Nvme->SqBuffer[0] = (NVME_SQ *)(UINTN)NVME_ASQ_BASE (Nvme);   // NVME_ADMIN_QUEUE
1730   Nvme->CqBuffer[0] = (NVME_CQ *)(UINTN)NVME_ACQ_BASE (Nvme);   // NVME_ADMIN_QUEUE
1731   Nvme->SqBuffer[1] = (NVME_SQ *)(UINTN)NVME_SQ_BASE (Nvme, 0); // NVME_IO_QUEUE
1732   Nvme->CqBuffer[1] = (NVME_CQ *)(UINTN)NVME_CQ_BASE (Nvme, 0); // NVME_IO_QUEUE
1733
1734   DEBUG ((DEBUG_INFO, "BaseMem = [%08X]\n", Nvme->BaseMem));
1735   DEBUG ((DEBUG_INFO, "Admin Submission Queue Size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));
1736   DEBUG ((DEBUG_INFO, "Admin Completion Queue Size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));
1737   DEBUG ((DEBUG_INFO, "Admin Submission Queue (SqBuffer[0]) =   [%08X]\n", Nvme->SqBuffer[0]));
1738   DEBUG ((DEBUG_INFO, "Admin Completion Queue (CqBuffer[0]) =   [%08X]\n", Nvme->CqBuffer[0]));
1739   DEBUG ((DEBUG_INFO, "I/O   Submission Queue (SqBuffer[1]) =   [%08X]\n", Nvme->SqBuffer[1]));
1740   DEBUG ((DEBUG_INFO, "I/O   Completion Queue (CqBuffer[1]) =   [%08X]\n", Nvme->CqBuffer[1]));
1741
1742   ///
1743   /// Program admin queue attributes.
1744   ///
1745   Status = NVME_SET_AQA (Nvme, &Aqa);
1746   if (EFI_ERROR(Status)) {
1747     goto Done;
1748   }
1749
1750   ///
1751   /// Program admin submission queue address.
1752   ///
1753   Status = NVME_SET_ASQ (Nvme, &Asq);
1754   if (EFI_ERROR(Status)) {
1755     goto Done;
1756   }
1757
1758   ///
1759   /// Program admin completion queue address.
1760   ///
1761   Status = NVME_SET_ACQ (Nvme, &Acq);
1762   if (EFI_ERROR(Status)) {
1763     goto Done;
1764   }
1765
1766   Status = NvmeEnableController (Nvme);
1767   if (EFI_ERROR(Status)) {
1768     goto Done;
1769   }
1770
1771   ///
1772   /// Create one I/O completion queue.
1773   ///
1774   Status = NvmeCreateIoCompletionQueue (Nvme);
1775   if (EFI_ERROR(Status)) {
1776     goto Done;
1777   }
1778
1779   ///
1780   /// Create one I/O Submission queue.
1781   ///
1782   Status = NvmeCreateIoSubmissionQueue (Nvme);
1783   if (EFI_ERROR(Status)) {
1784     goto Done;
1785   }
1786
1787   ///
1788   /// Get current Identify Controller Data
1789   ///
1790   Nvme->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)(UINTN) NVME_CONTROL_DATA_BASE (Nvme);
1791   Status = NvmeIdentifyController (Nvme, Nvme->ControllerData);
1792   if (EFI_ERROR(Status)) {
1793     goto Done;
1794   }
1795
1796   ///
1797   /// Dump NvmExpress Identify Controller Data
1798   ///
1799   Nvme->ControllerData->Sn[19] = 0;
1800   Nvme->ControllerData->Mn[39] = 0;
1801   //NvmeDumpIdentifyController (Nvme->ControllerData);
1802
1803   ///
1804   /// Get current Identify Namespace Data
1805   ///
1806   Nvme->NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)NVME_NAMESPACE_DATA_BASE (Nvme);
1807   Status = NvmeIdentifyNamespace (Nvme, Nvme->Nsid, Nvme->NamespaceData);
1808   if (EFI_ERROR(Status)) {
1809     DEBUG ((DEBUG_ERROR, "NvmeIdentifyNamespace fail, Status = %r\n", Status));
1810     goto Done;
1811   }
1812
1813   ///
1814   /// Dump NvmExpress Identify Namespace Data
1815   ///
1816   if (Nvme->NamespaceData->Ncap == 0) {
1817     DEBUG ((DEBUG_ERROR, "Invalid Namespace, Ncap: %lx\n", Nvme->NamespaceData->Ncap));
1818     Status = EFI_DEVICE_ERROR;
1819     goto Done;
1820   }
1821
1822   Nvme->BlockSize = NvmeGetBlockSize (Nvme);
1823   Nvme->LastBlock = NvmeGetLastLba (Nvme);
1824
1825   Nvme->State    = NvmeStatusInit;
1826
1827   return EFI_SUCCESS;
1828
1829 Done:
1830   return Status;
1831 }
1832
1833 /**
1834   Un-initialize the Nvm Express controller.
1835
1836   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1837
1838   @retval EFI_SUCCESS               - The NVM Express Controller is un-initialized successfully.
1839   @retval Others                    - A device error occurred while un-initializing the controller.
1840
1841 **/
1842 EFI_STATUS
1843 NvmeControllerExit (
1844   IN NVME_CONTEXT       *Nvme
1845   )
1846 {
1847   EFI_STATUS            Status;
1848
1849   Status = EFI_SUCCESS;
1850   if (Nvme->State == NvmeStatusInit || Nvme->State == NvmeStatusMax) {
1851     ///
1852     /// Destroy I/O Submission queue.
1853     ///
1854     Status = NvmeDestroyIoSubmissionQueue (Nvme);
1855     if (EFI_ERROR(Status)) {
1856       DEBUG ((DEBUG_ERROR, "NvmeDestroyIoSubmissionQueue fail, Status = %r\n", Status));
1857       return Status;
1858     }
1859
1860     ///
1861     /// Destroy I/O completion queue.
1862     ///
1863     Status = NvmeDestroyIoCompletionQueue (Nvme);
1864     if (EFI_ERROR(Status)) {
1865       DEBUG ((DEBUG_ERROR, "NvmeDestroyIoCompletionQueue fail, Status = %r\n", Status));
1866       return Status;
1867     }
1868
1869     Status = NvmeShutdownController (Nvme);
1870     if (EFI_ERROR(Status)) {
1871       DEBUG ((DEBUG_ERROR, "NvmeShutdownController fail, Status: %r\n", Status));
1872     }
1873   }
1874
1875   ///
1876   /// Disable PCIE decode
1877   ///
1878   PciWrite8  (Nvme->PciBase + NVME_PCIE_PCICMD, 0x0);
1879   PciWrite32 (Nvme->PciBase + 0x10, 0); // MLBAR (BAR0)
1880   PciWrite32 (Nvme->PciBase + 0x14, 0); // MUBAR (BAR1)
1881
1882   Nvme->State = NvmeStatusUnknown;
1883   return Status;
1884 }
1885
1886 /**
1887   Read sector Data from the NVMe device.
1888
1889   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1890   @param[in,out] Buffer             - The Buffer used to store the Data read from the device.
1891   @param[in] Lba                    - The start block number.
1892   @param[in] Blocks                 - Total block number to be read.
1893
1894   @retval EFI_SUCCESS               - Datum are read from the device.
1895   @retval Others                    - Fail to read all the datum.
1896
1897 **/
1898 EFI_STATUS
1899 NvmeReadSectors (
1900   IN NVME_CONTEXT                          *Nvme,
1901   IN OUT UINT64                            Buffer,
1902   IN UINT64                                Lba,
1903   IN UINT32                                Blocks
1904   )
1905 {
1906   UINT32                                   Bytes;
1907   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1908   NVM_EXPRESS_COMMAND                      Command;
1909   NVM_EXPRESS_RESPONSE                     Response;
1910   EFI_STATUS                               Status;
1911   UINT32                                   BlockSize;
1912
1913   BlockSize  = Nvme->BlockSize;
1914   Bytes      = Blocks * BlockSize;
1915
1916   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1917   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1918   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1919
1920   CommandPacket.NvmeCmd      = &Command;
1921   CommandPacket.NvmeResponse = &Response;
1922
1923   CommandPacket.NvmeCmd->Cdw0.Opcode = NVME_IO_READ_OPC;
1924   CommandPacket.NvmeCmd->Cdw0.Cid    = Nvme->Cid[NVME_IO_QUEUE]++;
1925   CommandPacket.NvmeCmd->Nsid        = Nvme->Nsid;
1926   CommandPacket.TransferBuffer       = Buffer;
1927
1928   CommandPacket.TransferLength = Bytes;
1929   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1930   CommandPacket.QueueId        = NVME_IO_QUEUE;
1931
1932   CommandPacket.NvmeCmd->Cdw10 = (UINT32)Lba;
1933   CommandPacket.NvmeCmd->Cdw11 = (UINT32)(RShiftU64 (Lba, 32));
1934   CommandPacket.NvmeCmd->Cdw12 = (Blocks - 1) & 0xFFFF;
1935
1936   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID | CDW12_VALID;
1937
1938   Status = NvmePassThru (
1939               Nvme,
1940               Nvme->Nsid,
1941               0,
1942               &CommandPacket
1943               );
1944
1945   return Status;
1946 }
1947
1948 /**
1949   Write sector Data to the NVMe device.
1950
1951   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1952   @param[in] Buffer                 - The Buffer to be written into the device.
1953   @param[in] Lba                    - The start block number.
1954   @param[in] Blocks                 - Total block number to be written.
1955
1956   @retval EFI_SUCCESS               - Datum are written into the Buffer.
1957   @retval Others                    - Fail to write all the datum.
1958
1959 **/
1960 EFI_STATUS
1961 NvmeWriteSectors (
1962   IN NVME_CONTEXT                          *Nvme,
1963   IN UINT64                                Buffer,
1964   IN UINT64                                Lba,
1965   IN UINT32                                Blocks
1966   )
1967 {
1968   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1969   NVM_EXPRESS_COMMAND                      Command;
1970   NVM_EXPRESS_RESPONSE                     Response;
1971   EFI_STATUS                               Status;
1972   UINT32                                   Bytes;
1973   UINT32                                   BlockSize;
1974
1975   BlockSize  = Nvme->BlockSize;
1976   Bytes      = Blocks * BlockSize;
1977
1978   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1979   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1980   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1981
1982   CommandPacket.NvmeCmd      = &Command;
1983   CommandPacket.NvmeResponse = &Response;
1984
1985   CommandPacket.NvmeCmd->Cdw0.Opcode = NVME_IO_WRITE_OPC;
1986   CommandPacket.NvmeCmd->Cdw0.Cid    = Nvme->Cid[NVME_IO_QUEUE]++;
1987   CommandPacket.NvmeCmd->Nsid  = Nvme->Nsid;
1988   CommandPacket.TransferBuffer = Buffer;
1989
1990   CommandPacket.TransferLength = Bytes;
1991   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1992   CommandPacket.QueueId        = NVME_IO_QUEUE;
1993
1994   CommandPacket.NvmeCmd->Cdw10 = (UINT32)Lba;
1995   CommandPacket.NvmeCmd->Cdw11 = (UINT32)(RShiftU64 (Lba, 32));
1996   CommandPacket.NvmeCmd->Cdw12 = (Blocks - 1) & 0xFFFF;
1997
1998   CommandPacket.MetadataBuffer = (UINT64)(UINTN)NULL;
1999   CommandPacket.MetadataLength = 0;
2000
2001   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID | CDW12_VALID;
2002
2003   Status = NvmePassThru (
2004               Nvme,
2005               Nvme->Nsid,
2006               0,
2007               &CommandPacket
2008               );
2009
2010   return Status;
2011 }
2012
2013 /**
2014   Flushes all modified Data to the device.
2015
2016   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
2017
2018   @retval EFI_SUCCESS               - Datum are written into the Buffer.
2019   @retval Others                    - Fail to write all the datum.
2020
2021 **/
2022 EFI_STATUS
2023 NvmeFlush (
2024   IN NVME_CONTEXT                          *Nvme
2025   )
2026 {
2027   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
2028   NVM_EXPRESS_COMMAND                      Command;
2029   NVM_EXPRESS_RESPONSE                     Response;
2030   EFI_STATUS                               Status;
2031
2032   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
2033   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
2034   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
2035
2036   CommandPacket.NvmeCmd      = &Command;
2037   CommandPacket.NvmeResponse = &Response;
2038
2039   CommandPacket.NvmeCmd->Cdw0.Opcode = NVME_IO_FLUSH_OPC;
2040   CommandPacket.NvmeCmd->Cdw0.Cid    = Nvme->Cid[NVME_IO_QUEUE]++;
2041   CommandPacket.NvmeCmd->Nsid  = Nvme->Nsid;
2042   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
2043   CommandPacket.QueueId        = NVME_IO_QUEUE;
2044
2045   Status = NvmePassThru (
2046               Nvme,
2047               Nvme->Nsid,
2048               0,
2049               &CommandPacket
2050               );
2051   if (!EFI_ERROR (Status)) {
2052     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
2053   }
2054
2055   return Status;
2056 }
2057
2058 /**
2059   Read some blocks from the device.
2060
2061   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
2062   @param[out] Buffer                - The Buffer used to store the Data read from the device.
2063   @param[in] Lba                    - The start block number.
2064   @param[in] Blocks                 - Total block number to be read.
2065
2066   @retval EFI_SUCCESS               - Datum are read from the device.
2067   @retval Others                    - Fail to read all the datum.
2068
2069 **/
2070 EFI_STATUS
2071 NvmeRead (
2072   IN NVME_CONTEXT                  *Nvme,
2073   OUT UINT64                       Buffer,
2074   IN UINT64                        Lba,
2075   IN UINTN                         Blocks
2076   )
2077 {
2078   EFI_STATUS                       Status;
2079   UINT32                           BlockSize;
2080   UINT32                           MaxTransferBlocks;
2081
2082   ASSERT (Blocks <= NVME_MAX_SECTORS);
2083   Status        = EFI_SUCCESS;
2084   BlockSize     = Nvme->BlockSize;
2085   if (Nvme->ControllerData->Mdts != 0) {
2086     MaxTransferBlocks = (1 << (Nvme->ControllerData->Mdts)) * (1 << (Nvme->Cap.Mpsmin + 12)) / BlockSize;
2087   } else {
2088     MaxTransferBlocks = 1024;
2089   }
2090
2091   while (Blocks > 0) {
2092     if (Blocks > MaxTransferBlocks) {
2093       Status = NvmeReadSectors (Nvme, Buffer, Lba, MaxTransferBlocks);
2094
2095       Blocks -= MaxTransferBlocks;
2096       Buffer += (MaxTransferBlocks * BlockSize);
2097       Lba    += MaxTransferBlocks;
2098     } else {
2099       Status = NvmeReadSectors (Nvme, Buffer, Lba, (UINT32) Blocks);
2100       Blocks = 0;
2101     }
2102
2103     if (EFI_ERROR(Status)) {
2104       DEBUG ((DEBUG_ERROR, "NvmeRead fail, Status = %r\n", Status));
2105       break;
2106     }
2107   }
2108
2109   return Status;
2110 }
2111
2112 /**
2113   Write some blocks to the device.
2114
2115   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
2116   @param[in] Buffer                 - The Buffer to be written into the device.
2117   @param[in] Lba                    - The start block number.
2118   @param[in] Blocks                 - Total block number to be written.
2119
2120   @retval EFI_SUCCESS               - Datum are written into the Buffer.
2121   @retval Others                    - Fail to write all the datum.
2122
2123 **/
2124 EFI_STATUS
2125 NvmeWrite (
2126   IN NVME_CONTEXT                  *Nvme,
2127   IN UINT64                        Buffer,
2128   IN UINT64                        Lba,
2129   IN UINTN                         Blocks
2130   )
2131 {
2132   EFI_STATUS                       Status;
2133   UINT32                           BlockSize;
2134   UINT32                           MaxTransferBlocks;
2135
2136   ASSERT (Blocks <= NVME_MAX_SECTORS);
2137   Status        = EFI_SUCCESS;
2138   BlockSize     = Nvme->BlockSize;
2139
2140   if (Nvme->ControllerData->Mdts != 0) {
2141     MaxTransferBlocks = (1 << (Nvme->ControllerData->Mdts)) * (1 << (Nvme->Cap.Mpsmin + 12)) / BlockSize;
2142   } else {
2143     MaxTransferBlocks = 1024;
2144   }
2145
2146   while (Blocks > 0) {
2147     if (Blocks > MaxTransferBlocks) {
2148       Status = NvmeWriteSectors (Nvme, Buffer, Lba, MaxTransferBlocks);
2149
2150       Blocks -= MaxTransferBlocks;
2151       Buffer += (MaxTransferBlocks * BlockSize);
2152       Lba    += MaxTransferBlocks;
2153     } else {
2154       Status = NvmeWriteSectors (Nvme, Buffer, Lba, (UINT32) Blocks);
2155       Blocks = 0;
2156     }
2157
2158     if (EFI_ERROR(Status)) {
2159       DEBUG ((DEBUG_ERROR, "NvmeWrite fail, Status = %r\n", Status));
2160       break;
2161     }
2162   }
2163
2164   return Status;
2165 }