SecurityPkg/Tcg/Opal/OpalPasswordSmm/OpalNvmeMode.c

   1 /** @file
   2   Provide functions to initialize NVME controller and perform NVME commands
   3
   4 Copyright (c) 2016, Intel Corporation. All rights reserved.<BR>
   5 This program and the accompanying materials
   6 are licensed and made available under the terms and conditions of the BSD License
   7 which accompanies this distribution.  The full text of the license may be found at
   8 http://opensource.org/licenses/bsd-license.php
   9
  10 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12
  13 **/
  14
  15 #include "OpalPasswordSmm.h"
  16
  17
  18 #define ALIGN(v, a)                         (UINTN)((((v) - 1) | ((a) - 1)) + 1)
  19
  20 ///
  21 /// NVME Host controller registers operation
  22 ///
  23 #define NVME_GET_CAP(Nvme, Cap)             NvmeMmioRead  (Cap, Nvme->Nbar + NVME_CAP_OFFSET, sizeof (NVME_CAP))
  24 #define NVME_GET_CC(Nvme, Cc)               NvmeMmioRead  (Cc, Nvme->Nbar + NVME_CC_OFFSET, sizeof (NVME_CC))
  25 #define NVME_SET_CC(Nvme, Cc)               NvmeMmioWrite (Nvme->Nbar + NVME_CC_OFFSET, Cc, sizeof (NVME_CC))
  26 #define NVME_GET_CSTS(Nvme, Csts)           NvmeMmioRead  (Csts, Nvme->Nbar + NVME_CSTS_OFFSET, sizeof (NVME_CSTS))
  27 #define NVME_GET_AQA(Nvme, Aqa)             NvmeMmioRead  (Aqa, Nvme->Nbar + NVME_AQA_OFFSET, sizeof (NVME_AQA))
  28 #define NVME_SET_AQA(Nvme, Aqa)             NvmeMmioWrite (Nvme->Nbar + NVME_AQA_OFFSET, Aqa, sizeof (NVME_AQA))
  29 #define NVME_GET_ASQ(Nvme, Asq)             NvmeMmioRead  (Asq, Nvme->Nbar + NVME_ASQ_OFFSET, sizeof (NVME_ASQ))
  30 #define NVME_SET_ASQ(Nvme, Asq)             NvmeMmioWrite (Nvme->Nbar + NVME_ASQ_OFFSET, Asq, sizeof (NVME_ASQ))
  31 #define NVME_GET_ACQ(Nvme, Acq)             NvmeMmioRead  (Acq, Nvme->Nbar + NVME_ACQ_OFFSET, sizeof (NVME_ACQ))
  32 #define NVME_SET_ACQ(Nvme, Acq)             NvmeMmioWrite (Nvme->Nbar + NVME_ACQ_OFFSET, Acq, sizeof (NVME_ACQ))
  33 #define NVME_GET_VER(Nvme, Ver)             NvmeMmioRead  (Ver, Nvme->Nbar + NVME_VER_OFFSET, sizeof (NVME_VER))
  34 #define NVME_SET_SQTDBL(Nvme, Qid, Sqtdbl)  NvmeMmioWrite (Nvme->Nbar + NVME_SQTDBL_OFFSET(Qid, Nvme->Cap.Dstrd), Sqtdbl, sizeof (NVME_SQTDBL))
  35 #define NVME_SET_CQHDBL(Nvme, Qid, Cqhdbl)  NvmeMmioWrite (Nvme->Nbar + NVME_CQHDBL_OFFSET(Qid, Nvme->Cap.Dstrd), Cqhdbl, sizeof (NVME_CQHDBL))
  36
  37 ///
  38 /// Base memory address
  39 ///
  40 enum {
  41   BASEMEM_CONTROLLER_DATA,
  42   BASEMEM_IDENTIFY_DATA,
  43   BASEMEM_ASQ,
  44   BASEMEM_ACQ,
  45   BASEMEM_SQ,
  46   BASEMEM_CQ,
  47   BASEMEM_PRP,
  48   BASEMEM_SECURITY,
  49   MAX_BASEMEM_COUNT
  50 };
  51
  52 ///
  53 /// All of base memories are 4K(0x1000) alignment
  54 ///
  55 #define NVME_MEM_BASE(Nvme)                 (Nvme->BaseMem)
  56 #define NVME_CONTROL_DATA_BASE(Nvme)        (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_CONTROLLER_DATA))                        * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  57 #define NVME_NAMESPACE_DATA_BASE(Nvme)      (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_IDENTIFY_DATA))                          * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  58 #define NVME_ASQ_BASE(Nvme)                 (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_ASQ))                                    * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  59 #define NVME_ACQ_BASE(Nvme)                 (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_ACQ))                                    * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  60 #define NVME_SQ_BASE(Nvme, index)           (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_SQ) + ((index)*(NVME_MAX_IO_QUEUES-1)))  * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  61 #define NVME_CQ_BASE(Nvme, index)           (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_CQ) + ((index)*(NVME_MAX_IO_QUEUES-1)))  * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  62 #define NVME_PRP_BASE(Nvme, index)          (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_PRP) + ((index)*NVME_PRP_SIZE))          * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  63 #define NVME_SEC_BASE(Nvme)                 (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_SECURITY))                               * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
  64
  65 /**
  66   Transfer MMIO Data to memory.
  67
  68   @param[in,out] MemBuffer - Destination: Memory address
  69   @param[in] MmioAddr      - Source: MMIO address
  70   @param[in] Size          - Size for read
  71
  72   @retval EFI_SUCCESS - MMIO read sucessfully
  73 **/
  74 EFI_STATUS
  75 NvmeMmioRead (
  76   IN OUT VOID *MemBuffer,
  77   IN     UINTN MmioAddr,
  78   IN     UINTN Size
  79   )
  80 {
  81   UINTN  Offset;
  82   UINT8  Data;
  83   UINT8  *Ptr;
  84
  85   // priority has adjusted
  86   switch (Size) {
  87     case 4:
  88       *((UINT32 *)MemBuffer) = MmioRead32 (MmioAddr);
  89       break;
  90
  91     case 8:
  92       *((UINT64 *)MemBuffer) = MmioRead64 (MmioAddr);
  93       break;
  94
  95     case 2:
  96       *((UINT16 *)MemBuffer) = MmioRead16 (MmioAddr);
  97       break;
  98
  99     case 1:
 100       *((UINT8 *)MemBuffer) = MmioRead8 (MmioAddr);
 101       break;
 102
 103     default:
 104       Ptr = (UINT8 *)MemBuffer;
 105       for (Offset = 0; Offset < Size; Offset += 1) {
 106         Data = MmioRead8 (MmioAddr + Offset);
 107         Ptr[Offset] = Data;
 108       }
 109       break;
 110   }
 111
 112   return EFI_SUCCESS;
 113 }
 114
 115 /**
 116   Transfer memory data to MMIO.
 117
 118   @param[in,out] MmioAddr - Destination: MMIO address
 119   @param[in] MemBuffer    - Source: Memory address
 120   @param[in] Size         - Size for write
 121
 122   @retval EFI_SUCCESS - MMIO write sucessfully
 123 **/
 124 EFI_STATUS
 125 NvmeMmioWrite (
 126   IN OUT UINTN MmioAddr,
 127   IN     VOID *MemBuffer,
 128   IN     UINTN Size
 129   )
 130 {
 131   UINTN  Offset;
 132   UINT8  Data;
 133   UINT8  *Ptr;
 134
 135   // priority has adjusted
 136   switch (Size) {
 137     case 4:
 138       MmioWrite32 (MmioAddr, *((UINT32 *)MemBuffer));
 139       break;
 140
 141     case 8:
 142       MmioWrite64 (MmioAddr, *((UINT64 *)MemBuffer));
 143       break;
 144
 145     case 2:
 146       MmioWrite16 (MmioAddr, *((UINT16 *)MemBuffer));
 147       break;
 148
 149     case 1:
 150       MmioWrite8 (MmioAddr, *((UINT8 *)MemBuffer));
 151       break;
 152
 153     default:
 154       Ptr = (UINT8 *)MemBuffer;
 155       for (Offset = 0; Offset < Size; Offset += 1) {
 156         Data = Ptr[Offset];
 157         MmioWrite8 (MmioAddr + Offset, Data);
 158       }
 159       break;
 160   }
 161
 162   return EFI_SUCCESS;
 163 }
 164
 165 /**
 166   Transfer MMIO data to memory.
 167
 168   @param[in,out] MemBuffer - Destination: Memory address
 169   @param[in] MmioAddr      - Source: MMIO address
 170   @param[in] Size          - Size for read
 171
 172   @retval EFI_SUCCESS - MMIO read sucessfully
 173 **/
 174 EFI_STATUS
 175 OpalPciRead (
 176   IN OUT VOID *MemBuffer,
 177   IN     UINTN MmioAddr,
 178   IN     UINTN Size
 179   )
 180 {
 181   UINTN  Offset;
 182   UINT8  Data;
 183   UINT8  *Ptr;
 184
 185   // priority has adjusted
 186   switch (Size) {
 187     case 4:
 188       *((UINT32 *)MemBuffer) = PciRead32 (MmioAddr);
 189       break;
 190
 191     case 2:
 192       *((UINT16 *)MemBuffer) = PciRead16 (MmioAddr);
 193       break;
 194
 195     case 1:
 196       *((UINT8 *)MemBuffer) = PciRead8 (MmioAddr);
 197       break;
 198
 199     default:
 200       Ptr = (UINT8 *)MemBuffer;
 201       for (Offset = 0; Offset < Size; Offset += 1) {
 202         Data = PciRead8 (MmioAddr + Offset);
 203         Ptr[Offset] = Data;
 204       }
 205       break;
 206   }
 207
 208   return EFI_SUCCESS;
 209 }
 210
 211 /**
 212   Transfer memory data to MMIO.
 213
 214   @param[in,out] MmioAddr - Destination: MMIO address
 215   @param[in] MemBuffer    - Source: Memory address
 216   @param[in] Size         - Size for write
 217
 218   @retval EFI_SUCCESS - MMIO write sucessfully
 219 **/
 220 EFI_STATUS
 221 OpalPciWrite (
 222   IN OUT UINTN MmioAddr,
 223   IN     VOID *MemBuffer,
 224   IN     UINTN Size
 225   )
 226 {
 227   UINTN  Offset;
 228   UINT8  Data;
 229   UINT8  *Ptr;
 230
 231   // priority has adjusted
 232   switch (Size) {
 233     case 4:
 234       PciWrite32 (MmioAddr, *((UINT32 *)MemBuffer));
 235       break;
 236
 237     case 2:
 238       PciWrite16 (MmioAddr, *((UINT16 *)MemBuffer));
 239       break;
 240
 241     case 1:
 242       PciWrite8 (MmioAddr, *((UINT8 *)MemBuffer));
 243       break;
 244
 245     default:
 246       Ptr = (UINT8 *)MemBuffer;
 247       for (Offset = 0; Offset < Size; Offset += 1) {
 248         Data = Ptr[Offset];
 249         PciWrite8 (MmioAddr + Offset, Data);
 250       }
 251       break;
 252   }
 253
 254   return EFI_SUCCESS;
 255 }
 256
 257 /**
 258   Get total pages for specific NVME based memory.
 259
 260   @param[in] BaseMemIndex           - The Index of BaseMem (0-based).
 261
 262   @retval - The page count for specific BaseMem Index
 263
 264 **/
 265 UINT32
 266 NvmeGetBaseMemPages (
 267   IN UINTN              BaseMemIndex
 268   )
 269 {
 270   UINT32                Pages;
 271   UINTN                 Index;
 272   UINT32                PageSizeList[8];
 273
 274   PageSizeList[0] = 1;  /* Controller Data */
 275   PageSizeList[1] = 1;  /* Identify Data */
 276   PageSizeList[2] = 1;  /* ASQ */
 277   PageSizeList[3] = 1;  /* ACQ */
 278   PageSizeList[4] = 1;  /* SQs */
 279   PageSizeList[5] = 1;  /* CQs */
 280   PageSizeList[6] = NVME_PRP_SIZE * NVME_CSQ_DEPTH;  /* PRPs */
 281   PageSizeList[7] = 1;  /* Security Commands */
 282
 283   if (BaseMemIndex > MAX_BASEMEM_COUNT) {
 284     ASSERT (FALSE);
 285     return 0;
 286   }
 287
 288   Pages = 0;
 289   for (Index = 0; Index < BaseMemIndex; Index++) {
 290     Pages += PageSizeList[Index];
 291   }
 292
 293   return Pages;
 294 }
 295
 296 /**
 297   Wait for NVME controller status to be ready or not.
 298
 299   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 300   @param[in] WaitReady              - Flag for waitting status ready or not
 301
 302   @return EFI_SUCCESS               - Successfully to wait specific status.
 303   @return others                    - Fail to wait for specific controller status.
 304
 305 **/
 306 STATIC
 307 EFI_STATUS
 308 NvmeWaitController (
 309   IN NVME_CONTEXT       *Nvme,
 310   IN BOOLEAN            WaitReady
 311   )
 312 {
 313   NVME_CSTS              Csts;
 314   EFI_STATUS             Status;
 315   UINT32                 Index;
 316   UINT8                  Timeout;
 317
 318   //
 319   // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
 320   // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
 321   //
 322   if (Nvme->Cap.To == 0) {
 323     Timeout = 1;
 324   } else {
 325     Timeout = Nvme->Cap.To;
 326   }
 327
 328   Status = EFI_SUCCESS;
 329   for(Index = (Timeout * 500); Index != 0; --Index) {
 330     MicroSecondDelay (1000);
 331
 332     //
 333     // Check if the controller is initialized
 334     //
 335     Status = NVME_GET_CSTS (Nvme, &Csts);
 336     if (EFI_ERROR(Status)) {
 337       DEBUG ((DEBUG_ERROR, "NVME_GET_CSTS fail, Status = %r\n", Status));
 338       return Status;
 339     }
 340
 341     if ((BOOLEAN) Csts.Rdy == WaitReady) {
 342       break;
 343     }
 344   }
 345
 346   if (Index == 0) {
 347     Status = EFI_TIMEOUT;
 348   }
 349
 350   return Status;
 351 }
 352
 353 /**
 354   Disable the Nvm Express controller.
 355
 356   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 357
 358   @return EFI_SUCCESS               - Successfully disable the controller.
 359   @return others                    - Fail to disable the controller.
 360
 361 **/
 362 STATIC
 363 EFI_STATUS
 364 NvmeDisableController (
 365   IN NVME_CONTEXT       *Nvme
 366   )
 367 {
 368   NVME_CC                Cc;
 369   NVME_CSTS              Csts;
 370   EFI_STATUS             Status;
 371
 372   Status = NVME_GET_CSTS (Nvme, &Csts);
 373
 374   ///
 375   /// Read Controller Configuration Register.
 376   ///
 377   Status = NVME_GET_CC (Nvme, &Cc);
 378   if (EFI_ERROR(Status)) {
 379     DEBUG ((DEBUG_ERROR, "NVME_GET_CC fail, Status = %r\n", Status));
 380     goto Done;
 381   }
 382
 383   if (Cc.En == 1) {
 384     Cc.En = 0;
 385     ///
 386     /// Disable the controller.
 387     ///
 388     Status = NVME_SET_CC (Nvme, &Cc);
 389     if (EFI_ERROR(Status)) {
 390       DEBUG ((DEBUG_ERROR, "NVME_SET_CC fail, Status = %r\n", Status));
 391       goto Done;
 392     }
 393   }
 394
 395   Status = NvmeWaitController (Nvme, FALSE);
 396   if (EFI_ERROR(Status)) {
 397     DEBUG ((DEBUG_ERROR, "NvmeWaitController fail, Status = %r\n", Status));
 398     goto Done;
 399   }
 400
 401   return EFI_SUCCESS;
 402
 403 Done:
 404   DEBUG ((DEBUG_INFO, "NvmeDisableController fail, Status: %r\n", Status));
 405   return Status;
 406 }
 407
 408 /**
 409   Enable the Nvm Express controller.
 410
 411   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 412
 413   @return EFI_SUCCESS               - Successfully enable the controller.
 414   @return EFI_DEVICE_ERROR          - Fail to enable the controller.
 415   @return EFI_TIMEOUT               - Fail to enable the controller in given time slot.
 416
 417 **/
 418 STATIC
 419 EFI_STATUS
 420 NvmeEnableController (
 421   IN NVME_CONTEXT       *Nvme
 422   )
 423 {
 424   NVME_CC                Cc;
 425   EFI_STATUS             Status;
 426
 427   //
 428   // Enable the controller
 429   //
 430   ZeroMem (&Cc, sizeof (NVME_CC));
 431   Cc.En     = 1;
 432   Cc.Iosqes = 6;
 433   Cc.Iocqes = 4;
 434   Status    = NVME_SET_CC (Nvme, &Cc);
 435   if (EFI_ERROR(Status)) {
 436     DEBUG ((DEBUG_ERROR, "NVME_SET_CC fail, Status = %r\n", Status));
 437     goto Done;
 438   }
 439
 440   Status = NvmeWaitController (Nvme, TRUE);
 441   if (EFI_ERROR(Status)) {
 442     DEBUG ((DEBUG_ERROR, "NvmeWaitController fail, Status = %r\n", Status));
 443     goto Done;
 444   }
 445
 446   return EFI_SUCCESS;
 447
 448 Done:
 449   DEBUG ((DEBUG_INFO, "NvmeEnableController fail, Status: %r\n", Status));
 450   return Status;
 451 }
 452
 453 /**
 454   Shutdown the Nvm Express controller.
 455
 456   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 457
 458   @return EFI_SUCCESS               - Successfully shutdown the controller.
 459   @return EFI_DEVICE_ERROR          - Fail to shutdown the controller.
 460   @return EFI_TIMEOUT               - Fail to shutdown the controller in given time slot.
 461
 462 **/
 463 STATIC
 464 EFI_STATUS
 465 NvmeShutdownController (
 466   IN NVME_CONTEXT       *Nvme
 467   )
 468 {
 469   NVME_CC                Cc;
 470   NVME_CSTS              Csts;
 471   EFI_STATUS             Status;
 472   UINT32                 Index;
 473   UINTN                  Timeout;
 474
 475   Status    = NVME_GET_CC (Nvme, &Cc);
 476   if (EFI_ERROR(Status)) {
 477     DEBUG ((DEBUG_ERROR, "NVME_GET_CC fail, Status = %r\n", Status));
 478     return Status;
 479   }
 480
 481   Cc.Shn     = 1; // Normal shutdown
 482
 483   Status    = NVME_SET_CC (Nvme, &Cc);
 484   if (EFI_ERROR(Status)) {
 485     DEBUG ((DEBUG_ERROR, "NVME_SET_CC fail, Status = %r\n", Status));
 486     return Status;
 487   }
 488
 489   Timeout = NVME_GENERIC_TIMEOUT/1000; // ms
 490   for(Index = (UINT32)(Timeout); Index != 0; --Index) {
 491     MicroSecondDelay (1000);
 492
 493     Status = NVME_GET_CSTS (Nvme, &Csts);
 494     if (EFI_ERROR(Status)) {
 495       DEBUG ((DEBUG_ERROR, "NVME_GET_CSTS fail, Status = %r\n", Status));
 496       return Status;
 497     }
 498
 499     if (Csts.Shst == 2) { // Shutdown processing complete
 500       break;
 501     }
 502   }
 503
 504   if (Index == 0) {
 505     Status = EFI_TIMEOUT;
 506   }
 507
 508   return Status;
 509 }
 510
 511 /**
 512   Check the execution status from a given completion queue entry.
 513
 514   @param[in]     Cq                 - A pointer to the NVME_CQ item.
 515
 516 **/
 517 EFI_STATUS
 518 NvmeCheckCqStatus (
 519   IN NVME_CQ             *Cq
 520   )
 521 {
 522   if (Cq->Sct == 0x0 && Cq->Sc == 0x0) {
 523     return EFI_SUCCESS;
 524   }
 525
 526   DEBUG ((DEBUG_INFO, "Dump NVMe Completion Entry Status from [0x%x]:\n", (UINTN)Cq));
 527   DEBUG ((DEBUG_INFO, "  SQ Identifier : [0x%x], Phase Tag : [%d], Cmd Identifier : [0x%x]\n", Cq->Sqid, Cq->Pt, Cq->Cid));
 528   DEBUG ((DEBUG_INFO, "  NVMe Cmd Execution Result - "));
 529
 530   switch (Cq->Sct) {
 531     case 0x0:
 532       switch (Cq->Sc) {
 533         case 0x0:
 534           DEBUG ((DEBUG_INFO, "Successful Completion\n"));
 535           return EFI_SUCCESS;
 536         case 0x1:
 537           DEBUG ((DEBUG_INFO, "Invalid Command Opcode\n"));
 538           break;
 539         case 0x2:
 540           DEBUG ((DEBUG_INFO, "Invalid Field in Command\n"));
 541           break;
 542         case 0x3:
 543           DEBUG ((DEBUG_INFO, "Command ID Conflict\n"));
 544           break;
 545         case 0x4:
 546           DEBUG ((DEBUG_INFO, "Data Transfer Error\n"));
 547           break;
 548         case 0x5:
 549           DEBUG ((DEBUG_INFO, "Commands Aborted due to Power Loss Notification\n"));
 550           break;
 551         case 0x6:
 552           DEBUG ((DEBUG_INFO, "Internal Device Error\n"));
 553           break;
 554         case 0x7:
 555           DEBUG ((DEBUG_INFO, "Command Abort Requested\n"));
 556           break;
 557         case 0x8:
 558           DEBUG ((DEBUG_INFO, "Command Aborted due to SQ Deletion\n"));
 559           break;
 560         case 0x9:
 561           DEBUG ((DEBUG_INFO, "Command Aborted due to Failed Fused Command\n"));
 562           break;
 563         case 0xA:
 564           DEBUG ((DEBUG_INFO, "Command Aborted due to Missing Fused Command\n"));
 565           break;
 566         case 0xB:
 567           DEBUG ((DEBUG_INFO, "Invalid Namespace or Format\n"));
 568           break;
 569         case 0xC:
 570           DEBUG ((DEBUG_INFO, "Command Sequence Error\n"));
 571           break;
 572         case 0xD:
 573           DEBUG ((DEBUG_INFO, "Invalid SGL Last Segment Descriptor\n"));
 574           break;
 575         case 0xE:
 576           DEBUG ((DEBUG_INFO, "Invalid Number of SGL Descriptors\n"));
 577           break;
 578         case 0xF:
 579           DEBUG ((DEBUG_INFO, "Data SGL Length Invalid\n"));
 580           break;
 581         case 0x10:
 582           DEBUG ((DEBUG_INFO, "Metadata SGL Length Invalid\n"));
 583           break;
 584         case 0x11:
 585           DEBUG ((DEBUG_INFO, "SGL Descriptor Type Invalid\n"));
 586           break;
 587         case 0x80:
 588           DEBUG ((DEBUG_INFO, "LBA Out of Range\n"));
 589           break;
 590         case 0x81:
 591           DEBUG ((DEBUG_INFO, "Capacity Exceeded\n"));
 592           break;
 593         case 0x82:
 594           DEBUG ((DEBUG_INFO, "Namespace Not Ready\n"));
 595           break;
 596         case 0x83:
 597           DEBUG ((DEBUG_INFO, "Reservation Conflict\n"));
 598           break;
 599       }
 600       break;
 601
 602     case 0x1:
 603       switch (Cq->Sc) {
 604         case 0x0:
 605           DEBUG ((DEBUG_INFO, "Completion Queue Invalid\n"));
 606           break;
 607         case 0x1:
 608           DEBUG ((DEBUG_INFO, "Invalid Queue Identifier\n"));
 609           break;
 610         case 0x2:
 611           DEBUG ((DEBUG_INFO, "Maximum Queue Size Exceeded\n"));
 612           break;
 613         case 0x3:
 614           DEBUG ((DEBUG_INFO, "Abort Command Limit Exceeded\n"));
 615           break;
 616         case 0x5:
 617           DEBUG ((DEBUG_INFO, "Asynchronous Event Request Limit Exceeded\n"));
 618           break;
 619         case 0x6:
 620           DEBUG ((DEBUG_INFO, "Invalid Firmware Slot\n"));
 621           break;
 622         case 0x7:
 623           DEBUG ((DEBUG_INFO, "Invalid Firmware Image\n"));
 624           break;
 625         case 0x8:
 626           DEBUG ((DEBUG_INFO, "Invalid Interrupt Vector\n"));
 627           break;
 628         case 0x9:
 629           DEBUG ((DEBUG_INFO, "Invalid Log Page\n"));
 630           break;
 631         case 0xA:
 632           DEBUG ((DEBUG_INFO, "Invalid Format\n"));
 633           break;
 634         case 0xB:
 635           DEBUG ((DEBUG_INFO, "Firmware Application Requires Conventional Reset\n"));
 636           break;
 637         case 0xC:
 638           DEBUG ((DEBUG_INFO, "Invalid Queue Deletion\n"));
 639           break;
 640         case 0xD:
 641           DEBUG ((DEBUG_INFO, "Feature Identifier Not Saveable\n"));
 642           break;
 643         case 0xE:
 644           DEBUG ((DEBUG_INFO, "Feature Not Changeable\n"));
 645           break;
 646         case 0xF:
 647           DEBUG ((DEBUG_INFO, "Feature Not Namespace Specific\n"));
 648           break;
 649         case 0x10:
 650           DEBUG ((DEBUG_INFO, "Firmware Application Requires NVM Subsystem Reset\n"));
 651           break;
 652         case 0x80:
 653           DEBUG ((DEBUG_INFO, "Conflicting Attributes\n"));
 654           break;
 655         case 0x81:
 656           DEBUG ((DEBUG_INFO, "Invalid Protection Information\n"));
 657           break;
 658         case 0x82:
 659           DEBUG ((DEBUG_INFO, "Attempted Write to Read Only Range\n"));
 660           break;
 661       }
 662       break;
 663
 664     case 0x2:
 665       switch (Cq->Sc) {
 666         case 0x80:
 667           DEBUG ((DEBUG_INFO, "Write Fault\n"));
 668           break;
 669         case 0x81:
 670           DEBUG ((DEBUG_INFO, "Unrecovered Read Error\n"));
 671           break;
 672         case 0x82:
 673           DEBUG ((DEBUG_INFO, "End-to-end Guard Check Error\n"));
 674           break;
 675         case 0x83:
 676           DEBUG ((DEBUG_INFO, "End-to-end Application Tag Check Error\n"));
 677           break;
 678         case 0x84:
 679           DEBUG ((DEBUG_INFO, "End-to-end Reference Tag Check Error\n"));
 680           break;
 681         case 0x85:
 682           DEBUG ((DEBUG_INFO, "Compare Failure\n"));
 683           break;
 684         case 0x86:
 685           DEBUG ((DEBUG_INFO, "Access Denied\n"));
 686           break;
 687       }
 688       break;
 689
 690     default:
 691       DEBUG ((DEBUG_INFO, "Unknown error\n"));
 692       break;
 693   }
 694
 695   return EFI_DEVICE_ERROR;
 696 }
 697
 698 /**
 699   Create PRP lists for Data transfer which is larger than 2 memory pages.
 700   Note here we calcuate the number of required PRP lists and allocate them at one time.
 701
 702   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 703   @param[in] SqId                   - The SQ index for this PRP
 704   @param[in] PhysicalAddr           - The physical base address of Data Buffer.
 705   @param[in] Pages                  - The number of pages to be transfered.
 706   @param[out] PrpListHost           - The host base address of PRP lists.
 707   @param[in,out] PrpListNo          - The number of PRP List.
 708
 709   @retval The pointer Value to the first PRP List of the PRP lists.
 710
 711 **/
 712 STATIC
 713 UINT64
 714 NvmeCreatePrpList (
 715   IN     NVME_CONTEXT                 *Nvme,
 716   IN     UINT16                       SqId,
 717   IN     EFI_PHYSICAL_ADDRESS         PhysicalAddr,
 718   IN     UINTN                        Pages,
 719      OUT VOID                         **PrpListHost,
 720   IN OUT UINTN                        *PrpListNo
 721   )
 722 {
 723   UINTN                       PrpEntryNo;
 724   UINT64                      PrpListBase;
 725   UINTN                       PrpListIndex;
 726   UINTN                       PrpEntryIndex;
 727   UINT64                      Remainder;
 728   EFI_PHYSICAL_ADDRESS        PrpListPhyAddr;
 729   UINTN                       Bytes;
 730   UINT8                       *PrpEntry;
 731   EFI_PHYSICAL_ADDRESS        NewPhyAddr;
 732
 733   ///
 734   /// The number of Prp Entry in a memory page.
 735   ///
 736   PrpEntryNo = EFI_PAGE_SIZE / sizeof (UINT64);
 737
 738   ///
 739   /// Calculate total PrpList number.
 740   ///
 741   *PrpListNo = (UINTN) DivU64x64Remainder ((UINT64)Pages, (UINT64)PrpEntryNo, &Remainder);
 742   if (Remainder != 0) {
 743     *PrpListNo += 1;
 744   }
 745
 746   if (*PrpListNo > NVME_PRP_SIZE) {
 747     DEBUG ((DEBUG_INFO, "NvmeCreatePrpList (PhysicalAddr: %lx, Pages: %x) PrpEntryNo: %x\n",
 748       PhysicalAddr, Pages, PrpEntryNo));
 749     DEBUG ((DEBUG_INFO, "*PrpListNo: %x, Remainder: %lx", *PrpListNo, Remainder));
 750     ASSERT (FALSE);
 751   }
 752   *PrpListHost = (VOID *)(UINTN) NVME_PRP_BASE (Nvme, SqId);
 753
 754   Bytes = EFI_PAGES_TO_SIZE (*PrpListNo);
 755   PrpListPhyAddr = (UINT64)(UINTN)(*PrpListHost);
 756
 757   ///
 758   /// Fill all PRP lists except of last one.
 759   ///
 760   ZeroMem (*PrpListHost, Bytes);
 761   for (PrpListIndex = 0; PrpListIndex < *PrpListNo - 1; ++PrpListIndex) {
 762     PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 763
 764     for (PrpEntryIndex = 0; PrpEntryIndex < PrpEntryNo; ++PrpEntryIndex) {
 765       PrpEntry = (UINT8 *)(UINTN) (PrpListBase + PrpEntryIndex * sizeof(UINT64));
 766       if (PrpEntryIndex != PrpEntryNo - 1) {
 767         ///
 768         /// Fill all PRP entries except of last one.
 769         ///
 770         CopyMem (PrpEntry, (VOID *)(UINTN) (&PhysicalAddr), sizeof (UINT64));
 771         PhysicalAddr += EFI_PAGE_SIZE;
 772       } else {
 773         ///
 774         /// Fill last PRP entries with next PRP List pointer.
 775         ///
 776         NewPhyAddr = (PrpListPhyAddr + (PrpListIndex + 1) * EFI_PAGE_SIZE);
 777         CopyMem (PrpEntry, (VOID *)(UINTN) (&NewPhyAddr), sizeof (UINT64));
 778       }
 779     }
 780   }
 781
 782   ///
 783   /// Fill last PRP list.
 784   ///
 785   PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 786   for (PrpEntryIndex = 0; PrpEntryIndex < ((Remainder != 0) ? Remainder : PrpEntryNo); ++PrpEntryIndex) {
 787     PrpEntry = (UINT8 *)(UINTN) (PrpListBase + PrpEntryIndex * sizeof(UINT64));
 788     CopyMem (PrpEntry, (VOID *)(UINTN) (&PhysicalAddr), sizeof (UINT64));
 789
 790     PhysicalAddr += EFI_PAGE_SIZE;
 791   }
 792
 793   return PrpListPhyAddr;
 794 }
 795
 796 /**
 797   Check whether there are available command slots.
 798
 799   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 800   @param[in] Qid                    - Queue index
 801
 802   @retval EFI_SUCCESS               - Available command slot is found
 803   @retval EFI_NOT_READY             - No available command slot is found
 804   @retval EFI_DEVICE_ERROR          - Error occurred on device side.
 805
 806 **/
 807 EFI_STATUS
 808 NvmeHasFreeCmdSlot (
 809   IN NVME_CONTEXT       *Nvme,
 810   IN UINT8              Qid
 811   )
 812 {
 813   return TRUE;
 814 }
 815
 816 /**
 817   Check whether all command slots are clean.
 818
 819   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 820   @param[in] Qid                    - Queue index
 821
 822   @retval EFI_SUCCESS               - All command slots are clean
 823   @retval EFI_NOT_READY             - Not all command slots are clean
 824   @retval EFI_DEVICE_ERROR          - Error occurred on device side.
 825
 826 **/
 827 EFI_STATUS
 828 NvmeIsAllCmdSlotClean (
 829   IN NVME_CONTEXT       *Nvme,
 830   IN UINT8              Qid
 831   )
 832 {
 833   return EFI_SUCCESS;
 834 }
 835
 836 /**
 837   Waits until all NVME commands completed.
 838
 839   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 840   @param[in] Qid                    - Queue index
 841
 842   @retval EFI_SUCCESS               - All NVME commands have completed
 843   @retval EFI_TIMEOUT               - Timeout occured
 844   @retval EFI_NOT_READY             - Not all NVME commands have completed
 845   @retval others                    - Error occurred on device side.
 846 **/
 847 EFI_STATUS
 848 NvmeWaitAllComplete (
 849   IN NVME_CONTEXT       *Nvme,
 850   IN UINT8              Qid
 851   )
 852 {
 853   return EFI_SUCCESS;
 854 }
 855
 856 /**
 857   Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function supports
 858   both blocking I/O and nonblocking I/O. The blocking I/O functionality is required, and the nonblocking
 859   I/O functionality is optional.
 860
 861   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
 862   @param[in] NamespaceId            - Is a 32 bit Namespace ID to which the Express HCI command packet will be sent.
 863                                       A Value of 0 denotes the NVM Express controller, a Value of all 0FFh in the namespace
 864                                       ID specifies that the command packet should be sent to all valid namespaces.
 865   @param[in] NamespaceUuid          - Is a 64 bit Namespace UUID to which the Express HCI command packet will be sent.
 866                                       A Value of 0 denotes the NVM Express controller, a Value of all 0FFh in the namespace
 867                                       UUID specifies that the command packet should be sent to all valid namespaces.
 868   @param[in,out] Packet             - A pointer to the NVM Express HCI Command Packet to send to the NVMe namespace specified
 869                                       by NamespaceId.
 870
 871   @retval EFI_SUCCESS               - The NVM Express Command Packet was sent by the host. TransferLength bytes were transferred
 872                                       to, or from DataBuffer.
 873   @retval EFI_NOT_READY             - The NVM Express Command Packet could not be sent because the controller is not ready. The caller
 874                                       may retry again later.
 875   @retval EFI_DEVICE_ERROR          - A device error occurred while attempting to send the NVM Express Command Packet.
 876   @retval EFI_INVALID_PARAMETER     - Namespace, or the contents of NVM_EXPRESS_PASS_THRU_COMMAND_PACKET are invalid. The NVM
 877                                       Express Command Packet was not sent, so no additional status information is available.
 878   @retval EFI_UNSUPPORTED           - The command described by the NVM Express Command Packet is not supported by the host adapter.
 879                                       The NVM Express Command Packet was not sent, so no additional status information is available.
 880   @retval EFI_TIMEOUT               - A timeout occurred while waiting for the NVM Express Command Packet to execute.
 881
 882 **/
 883 EFI_STATUS
 884 NvmePassThru (
 885   IN     NVME_CONTEXT                         *Nvme,
 886   IN     UINT32                               NamespaceId,
 887   IN     UINT64                               NamespaceUuid,
 888   IN OUT NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *Packet
 889   )
 890 {
 891   EFI_STATUS                    Status;
 892   NVME_SQ                       *Sq;
 893   NVME_CQ                       *Cq;
 894   UINT8                         Qid;
 895   UINT32                        Bytes;
 896   UINT32                        Offset;
 897   EFI_PHYSICAL_ADDRESS          PhyAddr;
 898   VOID                          *PrpListHost;
 899   UINTN                         PrpListNo;
 900   UINT32                        Timer;
 901   UINTN SqSize;
 902   UINTN CqSize;
 903
 904   ///
 905   /// check the Data fields in Packet parameter.
 906   ///
 907   if ((Nvme == NULL) || (Packet == NULL)) {
 908     DEBUG ((DEBUG_ERROR, "NvmePassThru, invalid parameter: Nvme(%x)/Packet(%x)\n",
 909       (UINTN)Nvme, (UINTN)Packet));
 910     return EFI_INVALID_PARAMETER;
 911   }
 912
 913   if ((Packet->NvmeCmd == NULL) || (Packet->NvmeResponse == NULL)) {
 914     DEBUG ((DEBUG_ERROR, "NvmePassThru, invalid parameter: NvmeCmd(%x)/NvmeResponse(%x)\n",
 915       (UINTN)Packet->NvmeCmd, (UINTN)Packet->NvmeResponse));
 916     return EFI_INVALID_PARAMETER;
 917   }
 918
 919   if (Packet->QueueId != NVME_ADMIN_QUEUE && Packet->QueueId != NVME_IO_QUEUE) {
 920     DEBUG ((DEBUG_ERROR, "NvmePassThru, invalid parameter: QueueId(%x)\n",
 921       Packet->QueueId));
 922     return EFI_INVALID_PARAMETER;
 923   }
 924
 925   PrpListHost = NULL;
 926   PrpListNo   = 0;
 927   Status      = EFI_SUCCESS;
 928
 929   Qid = Packet->QueueId;
 930   Sq  = Nvme->SqBuffer[Qid] + Nvme->SqTdbl[Qid].Sqt;
 931   Cq  = Nvme->CqBuffer[Qid] + Nvme->CqHdbl[Qid].Cqh;
 932   if (Qid == NVME_ADMIN_QUEUE) {
 933     SqSize = NVME_ASQ_SIZE + 1;
 934     CqSize = NVME_ACQ_SIZE + 1;
 935   } else {
 936     SqSize = NVME_CSQ_DEPTH;
 937     CqSize = NVME_CCQ_DEPTH;
 938   }
 939
 940   if (Packet->NvmeCmd->Nsid != NamespaceId) {
 941     DEBUG ((DEBUG_ERROR, "NvmePassThru: Nsid mismatch (%x, %x)\n",
 942       Packet->NvmeCmd->Nsid, NamespaceId));
 943     return EFI_INVALID_PARAMETER;
 944   }
 945
 946   ZeroMem (Sq, sizeof (NVME_SQ));
 947   Sq->Opc  = Packet->NvmeCmd->Cdw0.Opcode;
 948   Sq->Fuse = Packet->NvmeCmd->Cdw0.FusedOperation;
 949   Sq->Cid  = Packet->NvmeCmd->Cdw0.Cid;
 950   Sq->Nsid = Packet->NvmeCmd->Nsid;
 951
 952   ///
 953   /// Currently we only support PRP for Data transfer, SGL is NOT supported.
 954   ///
 955   ASSERT (Sq->Psdt == 0);
 956   if (Sq->Psdt != 0) {
 957     DEBUG ((DEBUG_ERROR, "NvmePassThru: doesn't support SGL mechanism\n"));
 958     return EFI_UNSUPPORTED;
 959   }
 960
 961   Sq->Prp[0] = Packet->TransferBuffer;
 962   Sq->Prp[1] = 0;
 963
 964   if(Packet->MetadataBuffer != (UINT64)(UINTN)NULL) {
 965     Sq->Mptr = Packet->MetadataBuffer;
 966   }
 967
 968   ///
 969   /// If the Buffer Size spans more than two memory pages (page Size as defined in CC.Mps),
 970   /// then build a PRP list in the second PRP submission queue entry.
 971   ///
 972   Offset = ((UINT32)Sq->Prp[0]) & (EFI_PAGE_SIZE - 1);
 973   Bytes  = Packet->TransferLength;
 974
 975   if ((Offset + Bytes) > (EFI_PAGE_SIZE * 2)) {
 976     ///
 977     /// Create PrpList for remaining Data Buffer.
 978     ///
 979     PhyAddr = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
 980     Sq->Prp[1] = NvmeCreatePrpList (Nvme, Nvme->SqTdbl[Qid].Sqt, PhyAddr, EFI_SIZE_TO_PAGES(Offset + Bytes) - 1, &PrpListHost, &PrpListNo);
 981     if (Sq->Prp[1] == 0) {
 982       Status = EFI_OUT_OF_RESOURCES;
 983       DEBUG ((DEBUG_ERROR, "NvmeCreatePrpList fail, Status: %r\n", Status));
 984       goto EXIT;
 985     }
 986
 987   } else if ((Offset + Bytes) > EFI_PAGE_SIZE) {
 988     Sq->Prp[1] = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
 989   }
 990
 991   if(Packet->NvmeCmd->Flags & CDW10_VALID) {
 992     Sq->Payload.Raw.Cdw10 = Packet->NvmeCmd->Cdw10;
 993   }
 994   if(Packet->NvmeCmd->Flags & CDW11_VALID) {
 995     Sq->Payload.Raw.Cdw11 = Packet->NvmeCmd->Cdw11;
 996   }
 997   if(Packet->NvmeCmd->Flags & CDW12_VALID) {
 998     Sq->Payload.Raw.Cdw12 = Packet->NvmeCmd->Cdw12;
 999   }
1000   if(Packet->NvmeCmd->Flags & CDW13_VALID) {
1001     Sq->Payload.Raw.Cdw13 = Packet->NvmeCmd->Cdw13;
1002   }
1003   if(Packet->NvmeCmd->Flags & CDW14_VALID) {
1004     Sq->Payload.Raw.Cdw14 = Packet->NvmeCmd->Cdw14;
1005   }
1006   if(Packet->NvmeCmd->Flags & CDW15_VALID) {
1007     Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
1008   }
1009
1010 #if (EN_NVME_VERBOSE_DBINFO == ON)
1011   //DumpMem (Sq, sizeof (NVME_SQ));
1012 #endif
1013   ///
1014   /// Ring the submission queue doorbell.
1015   ///
1016   Nvme->SqTdbl[Qid].Sqt++;
1017   if(Nvme->SqTdbl[Qid].Sqt == SqSize) {
1018     Nvme->SqTdbl[Qid].Sqt = 0;
1019   }
1020   Status = NVME_SET_SQTDBL (Nvme, Qid, &Nvme->SqTdbl[Qid]);
1021   if (EFI_ERROR(Status)) {
1022     DEBUG ((DEBUG_ERROR, "NVME_SET_SQTDBL fail, Status: %r\n", Status));
1023     goto EXIT;
1024   }
1025
1026   ///
1027   /// Wait for completion queue to get filled in.
1028   ///
1029   Status = EFI_TIMEOUT;
1030   Timer   = 0;
1031   while (Timer < NVME_CMD_TIMEOUT) {
1032     //DEBUG ((DEBUG_VERBOSE, "Timer: %x, Cq:\n", Timer));
1033     //DumpMem (Cq, sizeof (NVME_CQ));
1034     if (Cq->Pt != Nvme->Pt[Qid]) {
1035       Status = EFI_SUCCESS;
1036       break;
1037     }
1038
1039     MicroSecondDelay (NVME_CMD_WAIT);
1040     Timer += NVME_CMD_WAIT;
1041   }
1042
1043   Nvme->CqHdbl[Qid].Cqh++;
1044   if (Nvme->CqHdbl[Qid].Cqh == CqSize) {
1045     Nvme->CqHdbl[Qid].Cqh = 0;
1046     Nvme->Pt[Qid] ^= 1;
1047   }
1048
1049   ///
1050   /// Copy the Respose Queue entry for this command to the callers response Buffer
1051   ///
1052   CopyMem (Packet->NvmeResponse, Cq, sizeof(NVM_EXPRESS_RESPONSE));
1053
1054   if (!EFI_ERROR(Status)) { // We still need to check CQ status if no timeout error occured
1055     Status = NvmeCheckCqStatus (Cq);
1056   }
1057   NVME_SET_CQHDBL (Nvme, Qid, &Nvme->CqHdbl[Qid]);
1058
1059 EXIT:
1060   return Status;
1061 }
1062
1063 /**
1064   Get identify controller Data.
1065
1066   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1067   @param[in] Buffer                 - The Buffer used to store the identify controller Data.
1068
1069   @return EFI_SUCCESS               - Successfully get the identify controller Data.
1070   @return others                    - Fail to get the identify controller Data.
1071
1072 **/
1073 STATIC
1074 EFI_STATUS
1075 NvmeIdentifyController (
1076   IN NVME_CONTEXT                          *Nvme,
1077   IN VOID                                  *Buffer
1078   )
1079 {
1080   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1081   NVM_EXPRESS_COMMAND                      Command;
1082   NVM_EXPRESS_RESPONSE                     Response;
1083   EFI_STATUS                               Status;
1084
1085   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1086   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1087   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1088
1089   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_OPC;
1090   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1091   //
1092   // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
1093   // For the Identify command, the Namespace Identifier is only used for the Namespace Data structure.
1094   //
1095   Command.Nsid        = 0;
1096
1097   CommandPacket.NvmeCmd        = &Command;
1098   CommandPacket.NvmeResponse   = &Response;
1099   CommandPacket.TransferBuffer = (UINT64)(UINTN)Buffer;
1100   CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA);
1101   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1102   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1103   //
1104   // Set bit 0 (Cns bit) to 1 to identify a controller
1105   //
1106   Command.Cdw10                = 1;
1107   Command.Flags                = CDW10_VALID;
1108
1109   Status = NvmePassThru (
1110               Nvme,
1111               NVME_CONTROLLER_ID,
1112               0,
1113               &CommandPacket
1114               );
1115   if (!EFI_ERROR (Status)) {
1116     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1117   }
1118
1119   return Status;
1120 }
1121
1122 /**
1123   Get specified identify namespace Data.
1124
1125   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1126   @param[in] NamespaceId            - The specified namespace identifier.
1127   @param[in] Buffer                 - The Buffer used to store the identify namespace Data.
1128
1129   @return EFI_SUCCESS               - Successfully get the identify namespace Data.
1130   @return others                    - Fail to get the identify namespace Data.
1131
1132 **/
1133 STATIC
1134 EFI_STATUS
1135 NvmeIdentifyNamespace (
1136   IN NVME_CONTEXT                          *Nvme,
1137   IN UINT32                                NamespaceId,
1138   IN VOID                                  *Buffer
1139   )
1140 {
1141   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1142   NVM_EXPRESS_COMMAND                      Command;
1143   NVM_EXPRESS_RESPONSE                     Response;
1144   EFI_STATUS                               Status;
1145
1146   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1147   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1148   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1149
1150   CommandPacket.NvmeCmd      = &Command;
1151   CommandPacket.NvmeResponse = &Response;
1152
1153   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_OPC;
1154   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1155   Command.Nsid        = NamespaceId;
1156   CommandPacket.TransferBuffer = (UINT64)(UINTN)Buffer;
1157   CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA);
1158   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1159   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1160   //
1161   // Set bit 0 (Cns bit) to 1 to identify a namespace
1162   //
1163   CommandPacket.NvmeCmd->Cdw10 = 0;
1164   CommandPacket.NvmeCmd->Flags = CDW10_VALID;
1165
1166   Status = NvmePassThru (
1167               Nvme,
1168               NamespaceId,
1169               0,
1170               &CommandPacket
1171               );
1172   if (!EFI_ERROR (Status)) {
1173     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1174   }
1175
1176   return Status;
1177 }
1178
1179 /**
1180   Get Block Size for specific namespace of NVME.
1181
1182   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1183
1184   @return                           - Block Size in bytes
1185
1186 **/
1187 STATIC
1188 UINT32
1189 NvmeGetBlockSize (
1190   IN NVME_CONTEXT       *Nvme
1191   )
1192 {
1193   UINT32                BlockSize;
1194   UINT32                Lbads;
1195   UINT32                Flbas;
1196   UINT32                LbaFmtIdx;
1197
1198   Flbas     = Nvme->NamespaceData->Flbas;
1199   LbaFmtIdx = Flbas & 3;
1200   Lbads     = Nvme->NamespaceData->LbaFormat[LbaFmtIdx].Lbads;
1201
1202   BlockSize = (UINT32)1 << Lbads;
1203   return BlockSize;
1204 }
1205
1206 /**
1207   Get last LBA for specific namespace of NVME.
1208
1209   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1210
1211   @return                           - Last LBA address
1212
1213 **/
1214 STATIC
1215 EFI_LBA
1216 NvmeGetLastLba (
1217   IN NVME_CONTEXT       *Nvme
1218   )
1219 {
1220   EFI_LBA               LastBlock;
1221   LastBlock = Nvme->NamespaceData->Nsze - 1;
1222   return LastBlock;
1223 }
1224
1225 /**
1226   Create io completion queue.
1227
1228   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1229
1230   @return EFI_SUCCESS               - Successfully create io completion queue.
1231   @return others                    - Fail to create io completion queue.
1232
1233 **/
1234 STATIC
1235 EFI_STATUS
1236 NvmeCreateIoCompletionQueue (
1237   IN     NVME_CONTEXT                      *Nvme
1238   )
1239 {
1240   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1241   NVM_EXPRESS_COMMAND                      Command;
1242   NVM_EXPRESS_RESPONSE                     Response;
1243   EFI_STATUS                               Status;
1244   NVME_ADMIN_CRIOCQ                        CrIoCq;
1245
1246   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1247   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1248   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1249   ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ));
1250
1251   CommandPacket.NvmeCmd      = &Command;
1252   CommandPacket.NvmeResponse = &Response;
1253
1254   Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_OPC;
1255   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1256   CommandPacket.TransferBuffer = (UINT64)(UINTN)Nvme->CqBuffer[NVME_IO_QUEUE];
1257   CommandPacket.TransferLength = EFI_PAGE_SIZE;
1258   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1259   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1260
1261   CrIoCq.Qid   = NVME_IO_QUEUE;
1262   CrIoCq.Qsize = NVME_CCQ_SIZE;
1263   CrIoCq.Pc    = 1;
1264   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
1265   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1266
1267   Status = NvmePassThru (
1268               Nvme,
1269               NVME_CONTROLLER_ID,
1270               0,
1271               &CommandPacket
1272               );
1273   if (!EFI_ERROR (Status)) {
1274     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1275   }
1276
1277   return Status;
1278 }
1279
1280 /**
1281   Create io submission queue.
1282
1283   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1284
1285   @return EFI_SUCCESS               - Successfully create io submission queue.
1286   @return others                    - Fail to create io submission queue.
1287
1288 **/
1289 STATIC
1290 EFI_STATUS
1291 NvmeCreateIoSubmissionQueue (
1292   IN NVME_CONTEXT                          *Nvme
1293   )
1294 {
1295   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1296   NVM_EXPRESS_COMMAND                      Command;
1297   NVM_EXPRESS_RESPONSE                     Response;
1298   EFI_STATUS                               Status;
1299   NVME_ADMIN_CRIOSQ                        CrIoSq;
1300
1301   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1302   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1303   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1304   ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ));
1305
1306   CommandPacket.NvmeCmd      = &Command;
1307   CommandPacket.NvmeResponse = &Response;
1308
1309   Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_OPC;
1310   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1311   CommandPacket.TransferBuffer = (UINT64)(UINTN)Nvme->SqBuffer[NVME_IO_QUEUE];
1312   CommandPacket.TransferLength = EFI_PAGE_SIZE;
1313   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1314   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1315
1316   CrIoSq.Qid   = NVME_IO_QUEUE;
1317   CrIoSq.Qsize = NVME_CSQ_SIZE;
1318   CrIoSq.Pc    = 1;
1319   CrIoSq.Cqid  = NVME_IO_QUEUE;
1320   CrIoSq.Qprio = 0;
1321   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
1322   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1323
1324   Status = NvmePassThru (
1325               Nvme,
1326               NVME_CONTROLLER_ID,
1327               0,
1328               &CommandPacket
1329               );
1330   if (!EFI_ERROR (Status)) {
1331     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1332   }
1333
1334   return Status;
1335 }
1336
1337 /**
1338   Security send and receive commands.
1339
1340   @param[in]     Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1341   @param[in]     SendCommand            - The flag to indicate the command type, TRUE for Send command and FALSE for receive command
1342   @param[in]     SecurityProtocol       - Security Protocol
1343   @param[in]     SpSpecific             - Security Protocol Specific
1344   @param[in]     TransferLength         - Transfer Length of Buffer (in bytes) - always a multiple of 512
1345   @param[in,out] TransferBuffer         - Address of Data to transfer
1346
1347   @return EFI_SUCCESS               - Successfully create io submission queue.
1348   @return others                    - Fail to send/receive commands.
1349
1350 **/
1351 EFI_STATUS
1352 NvmeSecuritySendReceive (
1353   IN NVME_CONTEXT                          *Nvme,
1354   IN BOOLEAN                               SendCommand,
1355   IN UINT8                                 SecurityProtocol,
1356   IN UINT16                                SpSpecific,
1357   IN UINTN                                 TransferLength,
1358   IN OUT VOID                              *TransferBuffer
1359   )
1360 {
1361   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1362   NVM_EXPRESS_COMMAND                      Command;
1363   NVM_EXPRESS_RESPONSE                     Response;
1364   EFI_STATUS                               Status;
1365   NVME_ADMIN_SECSEND                       SecSend;
1366   OACS                                     *Oacs;
1367   UINT8                                    Opcode;
1368   VOID*                                    *SecBuff;
1369
1370   Oacs = (OACS *)&Nvme->ControllerData->Oacs;
1371
1372   //
1373   // Verify security bit for Security Send/Receive commands
1374   //
1375   if (Oacs->Security == 0) {
1376     DEBUG ((DEBUG_ERROR, "Security command doesn't support.\n"));
1377     return EFI_NOT_READY;
1378   }
1379
1380   SecBuff = (VOID *)(UINTN) NVME_SEC_BASE (Nvme);
1381
1382   //
1383   // Actions for sending security command
1384   //
1385   if (SendCommand) {
1386 #if (EN_NVME_VERBOSE_DBINFO == ON)
1387     //DumpMem (TransferBuffer, TransferLength);
1388 #endif
1389     CopyMem (SecBuff, TransferBuffer, TransferLength);
1390   }
1391
1392   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1393   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1394   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1395   ZeroMem (&SecSend, sizeof(NVME_ADMIN_SECSEND));
1396
1397   CommandPacket.NvmeCmd      = &Command;
1398   CommandPacket.NvmeResponse = &Response;
1399
1400   Opcode = (UINT8)(SendCommand ? NVME_ADMIN_SECURITY_SEND_OPC : NVME_ADMIN_SECURITY_RECV_OPC);
1401   Command.Cdw0.Opcode = Opcode;
1402   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1403   CommandPacket.TransferBuffer = (UINT64)(UINTN)SecBuff;
1404   CommandPacket.TransferLength = (UINT32)TransferLength;
1405   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1406   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1407
1408   SecSend.Spsp = SpSpecific;
1409   SecSend.Secp = SecurityProtocol;
1410   SecSend.Tl   = (UINT32)TransferLength;
1411
1412   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &SecSend, sizeof (NVME_ADMIN_SECSEND));
1413   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1414
1415   Status = NvmePassThru (
1416               Nvme,
1417               NVME_CONTROLLER_ID,
1418               0,
1419               &CommandPacket
1420               );
1421   if (!EFI_ERROR (Status)) {
1422     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1423   }
1424
1425   //
1426   // Actions for receiving security command
1427   //
1428   if (!SendCommand) {
1429     CopyMem (TransferBuffer, SecBuff, TransferLength);
1430 #if (EN_NVME_VERBOSE_DBINFO == ON)
1431     //DumpMem (TransferBuffer, TransferLength);
1432 #endif
1433   }
1434
1435   return Status;
1436 }
1437
1438 /**
1439   Destroy io completion queue.
1440
1441   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1442
1443   @return EFI_SUCCESS               - Successfully destroy io completion queue.
1444   @return others                    - Fail to destroy io completion queue.
1445
1446 **/
1447 STATIC
1448 EFI_STATUS
1449 NvmeDestroyIoCompletionQueue (
1450   IN     NVME_CONTEXT                      *Nvme
1451   )
1452 {
1453   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1454   NVM_EXPRESS_COMMAND                      Command;
1455   NVM_EXPRESS_RESPONSE                     Response;
1456   EFI_STATUS                               Status;
1457   NVME_ADMIN_DEIOCQ                        DelIoCq;
1458
1459   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1460   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1461   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1462   ZeroMem (&DelIoCq, sizeof(NVME_ADMIN_DEIOCQ));
1463
1464   CommandPacket.NvmeCmd      = &Command;
1465   CommandPacket.NvmeResponse = &Response;
1466
1467   Command.Cdw0.Opcode = NVME_ADMIN_DELIOCQ_OPC;
1468   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1469   CommandPacket.TransferBuffer = (UINT64)(UINTN)Nvme->CqBuffer[NVME_IO_QUEUE];
1470   CommandPacket.TransferLength = EFI_PAGE_SIZE;
1471   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1472   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1473
1474   DelIoCq.Qid   = NVME_IO_QUEUE;
1475   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &DelIoCq, sizeof (NVME_ADMIN_DEIOCQ));
1476   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1477
1478   Status = NvmePassThru (
1479               Nvme,
1480               NVME_CONTROLLER_ID,
1481               0,
1482               &CommandPacket
1483               );
1484   if (!EFI_ERROR (Status)) {
1485     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1486   }
1487
1488   return Status;
1489 }
1490
1491 /**
1492   Destroy io submission queue.
1493
1494   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1495
1496   @return EFI_SUCCESS               - Successfully destroy io submission queue.
1497   @return others                    - Fail to destroy io submission queue.
1498
1499 **/
1500 STATIC
1501 EFI_STATUS
1502 NvmeDestroyIoSubmissionQueue (
1503   IN NVME_CONTEXT                          *Nvme
1504   )
1505 {
1506   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1507   NVM_EXPRESS_COMMAND                      Command;
1508   NVM_EXPRESS_RESPONSE                     Response;
1509   EFI_STATUS                               Status;
1510   NVME_ADMIN_DEIOSQ                        DelIoSq;
1511
1512   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1513   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1514   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1515   ZeroMem (&DelIoSq, sizeof(NVME_ADMIN_DEIOSQ));
1516
1517   CommandPacket.NvmeCmd      = &Command;
1518   CommandPacket.NvmeResponse = &Response;
1519
1520   Command.Cdw0.Opcode = NVME_ADMIN_DELIOSQ_OPC;
1521   Command.Cdw0.Cid    = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1522   CommandPacket.TransferBuffer = (UINT64)(UINTN)Nvme->SqBuffer[NVME_IO_QUEUE];
1523   CommandPacket.TransferLength = EFI_PAGE_SIZE;
1524   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1525   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
1526
1527   DelIoSq.Qid   = NVME_IO_QUEUE;
1528   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &DelIoSq, sizeof (NVME_ADMIN_DEIOSQ));
1529   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1530
1531   Status = NvmePassThru (
1532               Nvme,
1533               NVME_CONTROLLER_ID,
1534               0,
1535               &CommandPacket
1536               );
1537   if (!EFI_ERROR (Status)) {
1538     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1539   }
1540
1541   return Status;
1542 }
1543
1544 /**
1545   Allocate transfer-related Data struct which is used at Nvme.
1546
1547   @param[in] ImageHandle         Image handle for this driver image
1548   @param[in] Nvme                The pointer to the NVME_CONTEXT Data structure.
1549
1550   @retval  EFI_OUT_OF_RESOURCE   The allocation is failure.
1551   @retval  EFI_SUCCESS           Successful to allocate memory.
1552
1553 **/
1554 EFI_STATUS
1555 EFIAPI
1556 NvmeAllocateResource (
1557   IN EFI_HANDLE                         ImageHandle,
1558   IN NVME_CONTEXT                       *Nvme
1559   )
1560 {
1561   EFI_STATUS            Status;
1562   EFI_PHYSICAL_ADDRESS  Addr;
1563   UINT32                Size;
1564
1565   //
1566   // Allocate resources required by NVMe host controller.
1567   //
1568   // NBAR
1569   Size = 0x10000;
1570   Addr = 0xFFFFFFFF;
1571   Status = gDS->AllocateMemorySpace (
1572                   EfiGcdAllocateMaxAddressSearchBottomUp,
1573                   EfiGcdMemoryTypeMemoryMappedIo,
1574                   15,                             // 2^15: 32K Alignment
1575                   Size,
1576                   &Addr,
1577                   ImageHandle,
1578                   NULL
1579                   );
1580   if (EFI_ERROR (Status)) {
1581     return EFI_OUT_OF_RESOURCES;
1582   }
1583   Nvme->Nbar = (UINT32) Addr;
1584
1585   // DMA Buffer
1586   Size = NVME_MEM_MAX_SIZE;
1587   Addr = 0xFFFFFFFF;
1588   Status = gBS->AllocatePages (
1589                   AllocateMaxAddress,
1590                   EfiACPIMemoryNVS,
1591                   EFI_SIZE_TO_PAGES (Size),
1592                   (EFI_PHYSICAL_ADDRESS *)&Addr
1593                   );
1594   if (EFI_ERROR (Status)) {
1595     return EFI_OUT_OF_RESOURCES;
1596   }
1597   Nvme->BaseMem = (UINT32) Addr;
1598
1599   // Clean up DMA Buffer before using
1600   ZeroMem ((VOID *)(UINTN)Addr, NVME_MEM_MAX_SIZE);
1601
1602   return EFI_SUCCESS;
1603 }
1604
1605 /**
1606   Free allocated transfer-related Data struct which is used at NVMe.
1607
1608   @param[in] Nvme                The pointer to the NVME_CONTEXT Data structure.
1609
1610 **/
1611 VOID
1612 EFIAPI
1613 NvmeFreeResource (
1614   IN NVME_CONTEXT                       *Nvme
1615   )
1616 {
1617   UINT32                Size;
1618
1619   // NBAR
1620   if (Nvme->BaseMem != 0) {
1621     Size = 0x10000;
1622     gDS->FreeMemorySpace (Nvme->Nbar, Size);
1623   }
1624
1625   // DMA Buffer
1626   if (Nvme->Nbar != 0) {
1627     Size = NVME_MEM_MAX_SIZE;
1628     gBS->FreePages ((EFI_PHYSICAL_ADDRESS)(UINTN) Nvme->Nbar, EFI_SIZE_TO_PAGES (Size));
1629   }
1630 }
1631
1632
1633 /**
1634   Initialize the Nvm Express controller.
1635
1636   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1637
1638   @retval EFI_SUCCESS               - The NVM Express Controller is initialized successfully.
1639   @retval Others                    - A device error occurred while initializing the controller.
1640
1641 **/
1642 EFI_STATUS
1643 NvmeControllerInit (
1644   IN NVME_CONTEXT       *Nvme
1645   )
1646 {
1647   EFI_STATUS            Status;
1648   NVME_AQA              Aqa;
1649   NVME_ASQ              Asq;
1650   NVME_ACQ              Acq;
1651   NVME_VER              Ver;
1652
1653   UINT32                MlBAR;
1654   UINT32                MuBAR;
1655
1656   ///
1657   /// Update PCIE BAR0/1 for NVME device
1658   ///
1659   MlBAR = Nvme->Nbar;
1660   MuBAR = 0;
1661   PciWrite32 (Nvme->PciBase + 0x10, MlBAR); // MLBAR (BAR0)
1662   PciWrite32 (Nvme->PciBase + 0x14, MuBAR); // MUBAR (BAR1)
1663
1664   ///
1665   /// Enable PCIE decode
1666   ///
1667   PciWrite8 (Nvme->PciBase + NVME_PCIE_PCICMD, 0x6);
1668
1669   // Version
1670   NVME_GET_VER (Nvme, &Ver);
1671   if (!(Ver.Mjr == 0x0001) && (Ver.Mnr == 0x0000)) {
1672     DEBUG ((DEBUG_INFO, "\n!!!\n!!! NVME Version mismatch for the implementation !!!\n!!!\n"));
1673   }
1674
1675   ///
1676   /// Read the Controller Capabilities register and verify that the NVM command set is supported
1677   ///
1678   Status = NVME_GET_CAP (Nvme, &Nvme->Cap);
1679   if (EFI_ERROR (Status)) {
1680     DEBUG ((DEBUG_ERROR, "NVME_GET_CAP fail, Status: %r\n", Status));
1681     goto Done;
1682   }
1683
1684   if (Nvme->Cap.Css != 0x01) {
1685     DEBUG ((DEBUG_ERROR, "NvmeControllerInit fail: the controller doesn't support NVMe command set\n"));
1686     Status = EFI_UNSUPPORTED;
1687     goto Done;
1688   }
1689
1690   ///
1691   /// Currently the driver only supports 4k page Size.
1692   ///
1693   if ((Nvme->Cap.Mpsmin + 12) > EFI_PAGE_SHIFT) {
1694     DEBUG ((DEBUG_ERROR, "NvmeControllerInit fail: only supports 4k page Size\n"));
1695     ASSERT (FALSE);
1696     Status = EFI_UNSUPPORTED;
1697     goto Done;
1698   }
1699
1700   Nvme->Cid[0] = 0;
1701   Nvme->Cid[1] = 0;
1702
1703   Status = NvmeDisableController (Nvme);
1704   if (EFI_ERROR(Status)) {
1705     DEBUG ((DEBUG_ERROR, "NvmeDisableController fail, Status: %r\n", Status));
1706     goto Done;
1707   }
1708
1709   ///
1710   /// set number of entries admin submission & completion queues.
1711   ///
1712   Aqa.Asqs  = NVME_ASQ_SIZE;
1713   Aqa.Rsvd1 = 0;
1714   Aqa.Acqs  = NVME_ACQ_SIZE;
1715   Aqa.Rsvd2 = 0;
1716
1717   ///
1718   /// Address of admin submission queue.
1719   ///
1720   Asq = (UINT64)(UINTN)(NVME_ASQ_BASE (Nvme) & ~0xFFF);
1721
1722   ///
1723   /// Address of admin completion queue.
1724   ///
1725   Acq = (UINT64)(UINTN)(NVME_ACQ_BASE (Nvme) & ~0xFFF);
1726
1727   ///
1728   /// Address of I/O submission & completion queue.
1729   ///
1730   Nvme->SqBuffer[0] = (NVME_SQ *)(UINTN)NVME_ASQ_BASE (Nvme);   // NVME_ADMIN_QUEUE
1731   Nvme->CqBuffer[0] = (NVME_CQ *)(UINTN)NVME_ACQ_BASE (Nvme);   // NVME_ADMIN_QUEUE
1732   Nvme->SqBuffer[1] = (NVME_SQ *)(UINTN)NVME_SQ_BASE (Nvme, 0); // NVME_IO_QUEUE
1733   Nvme->CqBuffer[1] = (NVME_CQ *)(UINTN)NVME_CQ_BASE (Nvme, 0); // NVME_IO_QUEUE
1734
1735   DEBUG ((DEBUG_INFO, "BaseMem = [%08X]\n", Nvme->BaseMem));
1736   DEBUG ((DEBUG_INFO, "Admin Submission Queue Size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));
1737   DEBUG ((DEBUG_INFO, "Admin Completion Queue Size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));
1738   DEBUG ((DEBUG_INFO, "Admin Submission Queue (SqBuffer[0]) =   [%08X]\n", Nvme->SqBuffer[0]));
1739   DEBUG ((DEBUG_INFO, "Admin Completion Queue (CqBuffer[0]) =   [%08X]\n", Nvme->CqBuffer[0]));
1740   DEBUG ((DEBUG_INFO, "I/O   Submission Queue (SqBuffer[1]) =   [%08X]\n", Nvme->SqBuffer[1]));
1741   DEBUG ((DEBUG_INFO, "I/O   Completion Queue (CqBuffer[1]) =   [%08X]\n", Nvme->CqBuffer[1]));
1742
1743   ///
1744   /// Program admin queue attributes.
1745   ///
1746   Status = NVME_SET_AQA (Nvme, &Aqa);
1747   if (EFI_ERROR(Status)) {
1748     goto Done;
1749   }
1750
1751   ///
1752   /// Program admin submission queue address.
1753   ///
1754   Status = NVME_SET_ASQ (Nvme, &Asq);
1755   if (EFI_ERROR(Status)) {
1756     goto Done;
1757   }
1758
1759   ///
1760   /// Program admin completion queue address.
1761   ///
1762   Status = NVME_SET_ACQ (Nvme, &Acq);
1763   if (EFI_ERROR(Status)) {
1764     goto Done;
1765   }
1766
1767   Status = NvmeEnableController (Nvme);
1768   if (EFI_ERROR(Status)) {
1769     goto Done;
1770   }
1771
1772   ///
1773   /// Create one I/O completion queue.
1774   ///
1775   Status = NvmeCreateIoCompletionQueue (Nvme);
1776   if (EFI_ERROR(Status)) {
1777     goto Done;
1778   }
1779
1780   ///
1781   /// Create one I/O Submission queue.
1782   ///
1783   Status = NvmeCreateIoSubmissionQueue (Nvme);
1784   if (EFI_ERROR(Status)) {
1785     goto Done;
1786   }
1787
1788   ///
1789   /// Get current Identify Controller Data
1790   ///
1791   Nvme->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)(UINTN) NVME_CONTROL_DATA_BASE (Nvme);
1792   Status = NvmeIdentifyController (Nvme, Nvme->ControllerData);
1793   if (EFI_ERROR(Status)) {
1794     goto Done;
1795   }
1796
1797   ///
1798   /// Dump NvmExpress Identify Controller Data
1799   ///
1800   Nvme->ControllerData->Sn[19] = 0;
1801   Nvme->ControllerData->Mn[39] = 0;
1802   //NvmeDumpIdentifyController (Nvme->ControllerData);
1803
1804   ///
1805   /// Get current Identify Namespace Data
1806   ///
1807   Nvme->NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)NVME_NAMESPACE_DATA_BASE (Nvme);
1808   Status = NvmeIdentifyNamespace (Nvme, Nvme->Nsid, Nvme->NamespaceData);
1809   if (EFI_ERROR(Status)) {
1810     DEBUG ((DEBUG_ERROR, "NvmeIdentifyNamespace fail, Status = %r\n", Status));
1811     goto Done;
1812   }
1813
1814   ///
1815   /// Dump NvmExpress Identify Namespace Data
1816   ///
1817   if (Nvme->NamespaceData->Ncap == 0) {
1818     DEBUG ((DEBUG_ERROR, "Invalid Namespace, Ncap: %lx\n", Nvme->NamespaceData->Ncap));
1819     Status = EFI_DEVICE_ERROR;
1820     goto Done;
1821   }
1822
1823   Nvme->BlockSize = NvmeGetBlockSize (Nvme);
1824   Nvme->LastBlock = NvmeGetLastLba (Nvme);
1825
1826   Nvme->State    = NvmeStatusInit;
1827
1828   return EFI_SUCCESS;
1829
1830 Done:
1831   return Status;
1832 }
1833
1834 /**
1835   Un-initialize the Nvm Express controller.
1836
1837   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1838
1839   @retval EFI_SUCCESS               - The NVM Express Controller is un-initialized successfully.
1840   @retval Others                    - A device error occurred while un-initializing the controller.
1841
1842 **/
1843 EFI_STATUS
1844 NvmeControllerExit (
1845   IN NVME_CONTEXT       *Nvme
1846   )
1847 {
1848   EFI_STATUS            Status;
1849
1850   Status = EFI_SUCCESS;
1851   if (Nvme->State == NvmeStatusInit || Nvme->State == NvmeStatusMax) {
1852     ///
1853     /// Destroy I/O Submission queue.
1854     ///
1855     Status = NvmeDestroyIoSubmissionQueue (Nvme);
1856     if (EFI_ERROR(Status)) {
1857       DEBUG ((DEBUG_ERROR, "NvmeDestroyIoSubmissionQueue fail, Status = %r\n", Status));
1858       return Status;
1859     }
1860
1861     ///
1862     /// Destroy I/O completion queue.
1863     ///
1864     Status = NvmeDestroyIoCompletionQueue (Nvme);
1865     if (EFI_ERROR(Status)) {
1866       DEBUG ((DEBUG_ERROR, "NvmeDestroyIoCompletionQueue fail, Status = %r\n", Status));
1867       return Status;
1868     }
1869
1870     Status = NvmeShutdownController (Nvme);
1871     if (EFI_ERROR(Status)) {
1872       DEBUG ((DEBUG_ERROR, "NvmeShutdownController fail, Status: %r\n", Status));
1873     }
1874   }
1875
1876   ///
1877   /// Disable PCIE decode
1878   ///
1879   PciWrite8  (Nvme->PciBase + NVME_PCIE_PCICMD, 0x0);
1880   PciWrite32 (Nvme->PciBase + 0x10, 0); // MLBAR (BAR0)
1881   PciWrite32 (Nvme->PciBase + 0x14, 0); // MUBAR (BAR1)
1882
1883   Nvme->State = NvmeStatusUnknown;
1884   return Status;
1885 }
1886
1887 /**
1888   Read sector Data from the NVMe device.
1889
1890   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1891   @param[in,out] Buffer             - The Buffer used to store the Data read from the device.
1892   @param[in] Lba                    - The start block number.
1893   @param[in] Blocks                 - Total block number to be read.
1894
1895   @retval EFI_SUCCESS               - Datum are read from the device.
1896   @retval Others                    - Fail to read all the datum.
1897
1898 **/
1899 EFI_STATUS
1900 NvmeReadSectors (
1901   IN NVME_CONTEXT                          *Nvme,
1902   IN OUT UINT64                            Buffer,
1903   IN UINT64                                Lba,
1904   IN UINT32                                Blocks
1905   )
1906 {
1907   UINT32                                   Bytes;
1908   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1909   NVM_EXPRESS_COMMAND                      Command;
1910   NVM_EXPRESS_RESPONSE                     Response;
1911   EFI_STATUS                               Status;
1912   UINT32                                   BlockSize;
1913
1914   BlockSize  = Nvme->BlockSize;
1915   Bytes      = Blocks * BlockSize;
1916
1917   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1918   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1919   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1920
1921   CommandPacket.NvmeCmd      = &Command;
1922   CommandPacket.NvmeResponse = &Response;
1923
1924   CommandPacket.NvmeCmd->Cdw0.Opcode = NVME_IO_READ_OPC;
1925   CommandPacket.NvmeCmd->Cdw0.Cid    = Nvme->Cid[NVME_IO_QUEUE]++;
1926   CommandPacket.NvmeCmd->Nsid        = Nvme->Nsid;
1927   CommandPacket.TransferBuffer       = Buffer;
1928
1929   CommandPacket.TransferLength = Bytes;
1930   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1931   CommandPacket.QueueId        = NVME_IO_QUEUE;
1932
1933   CommandPacket.NvmeCmd->Cdw10 = (UINT32)Lba;
1934   CommandPacket.NvmeCmd->Cdw11 = (UINT32)(RShiftU64 (Lba, 32));
1935   CommandPacket.NvmeCmd->Cdw12 = (Blocks - 1) & 0xFFFF;
1936
1937   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID | CDW12_VALID;
1938
1939   Status = NvmePassThru (
1940               Nvme,
1941               Nvme->Nsid,
1942               0,
1943               &CommandPacket
1944               );
1945
1946   return Status;
1947 }
1948
1949 /**
1950   Write sector Data to the NVMe device.
1951
1952   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
1953   @param[in] Buffer                 - The Buffer to be written into the device.
1954   @param[in] Lba                    - The start block number.
1955   @param[in] Blocks                 - Total block number to be written.
1956
1957   @retval EFI_SUCCESS               - Datum are written into the Buffer.
1958   @retval Others                    - Fail to write all the datum.
1959
1960 **/
1961 EFI_STATUS
1962 NvmeWriteSectors (
1963   IN NVME_CONTEXT                          *Nvme,
1964   IN UINT64                                Buffer,
1965   IN UINT64                                Lba,
1966   IN UINT32                                Blocks
1967   )
1968 {
1969   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
1970   NVM_EXPRESS_COMMAND                      Command;
1971   NVM_EXPRESS_RESPONSE                     Response;
1972   EFI_STATUS                               Status;
1973   UINT32                                   Bytes;
1974   UINT32                                   BlockSize;
1975
1976   BlockSize  = Nvme->BlockSize;
1977   Bytes      = Blocks * BlockSize;
1978
1979   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1980   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1981   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1982
1983   CommandPacket.NvmeCmd      = &Command;
1984   CommandPacket.NvmeResponse = &Response;
1985
1986   CommandPacket.NvmeCmd->Cdw0.Opcode = NVME_IO_WRITE_OPC;
1987   CommandPacket.NvmeCmd->Cdw0.Cid    = Nvme->Cid[NVME_IO_QUEUE]++;
1988   CommandPacket.NvmeCmd->Nsid  = Nvme->Nsid;
1989   CommandPacket.TransferBuffer = Buffer;
1990
1991   CommandPacket.TransferLength = Bytes;
1992   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1993   CommandPacket.QueueId        = NVME_IO_QUEUE;
1994
1995   CommandPacket.NvmeCmd->Cdw10 = (UINT32)Lba;
1996   CommandPacket.NvmeCmd->Cdw11 = (UINT32)(RShiftU64 (Lba, 32));
1997   CommandPacket.NvmeCmd->Cdw12 = (Blocks - 1) & 0xFFFF;
1998
1999   CommandPacket.MetadataBuffer = (UINT64)(UINTN)NULL;
2000   CommandPacket.MetadataLength = 0;
2001
2002   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID | CDW12_VALID;
2003
2004   Status = NvmePassThru (
2005               Nvme,
2006               Nvme->Nsid,
2007               0,
2008               &CommandPacket
2009               );
2010
2011   return Status;
2012 }
2013
2014 /**
2015   Flushes all modified Data to the device.
2016
2017   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
2018
2019   @retval EFI_SUCCESS               - Datum are written into the Buffer.
2020   @retval Others                    - Fail to write all the datum.
2021
2022 **/
2023 EFI_STATUS
2024 NvmeFlush (
2025   IN NVME_CONTEXT                          *Nvme
2026   )
2027 {
2028   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
2029   NVM_EXPRESS_COMMAND                      Command;
2030   NVM_EXPRESS_RESPONSE                     Response;
2031   EFI_STATUS                               Status;
2032
2033   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
2034   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
2035   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
2036
2037   CommandPacket.NvmeCmd      = &Command;
2038   CommandPacket.NvmeResponse = &Response;
2039
2040   CommandPacket.NvmeCmd->Cdw0.Opcode = NVME_IO_FLUSH_OPC;
2041   CommandPacket.NvmeCmd->Cdw0.Cid    = Nvme->Cid[NVME_IO_QUEUE]++;
2042   CommandPacket.NvmeCmd->Nsid  = Nvme->Nsid;
2043   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
2044   CommandPacket.QueueId        = NVME_IO_QUEUE;
2045
2046   Status = NvmePassThru (
2047               Nvme,
2048               Nvme->Nsid,
2049               0,
2050               &CommandPacket
2051               );
2052   if (!EFI_ERROR (Status)) {
2053     Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
2054   }
2055
2056   return Status;
2057 }
2058
2059 /**
2060   Read some blocks from the device.
2061
2062   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
2063   @param[out] Buffer                - The Buffer used to store the Data read from the device.
2064   @param[in] Lba                    - The start block number.
2065   @param[in] Blocks                 - Total block number to be read.
2066
2067   @retval EFI_SUCCESS               - Datum are read from the device.
2068   @retval Others                    - Fail to read all the datum.
2069
2070 **/
2071 EFI_STATUS
2072 NvmeRead (
2073   IN NVME_CONTEXT                  *Nvme,
2074   OUT UINT64                       Buffer,
2075   IN UINT64                        Lba,
2076   IN UINTN                         Blocks
2077   )
2078 {
2079   EFI_STATUS                       Status;
2080   UINT32                           BlockSize;
2081   UINT32                           MaxTransferBlocks;
2082
2083   ASSERT (Blocks <= NVME_MAX_SECTORS);
2084   Status        = EFI_SUCCESS;
2085   BlockSize     = Nvme->BlockSize;
2086   if (Nvme->ControllerData->Mdts != 0) {
2087     MaxTransferBlocks = (1 << (Nvme->ControllerData->Mdts)) * (1 << (Nvme->Cap.Mpsmin + 12)) / BlockSize;
2088   } else {
2089     MaxTransferBlocks = 1024;
2090   }
2091
2092   while (Blocks > 0) {
2093     if (Blocks > MaxTransferBlocks) {
2094       Status = NvmeReadSectors (Nvme, Buffer, Lba, MaxTransferBlocks);
2095
2096       Blocks -= MaxTransferBlocks;
2097       Buffer += (MaxTransferBlocks * BlockSize);
2098       Lba    += MaxTransferBlocks;
2099     } else {
2100       Status = NvmeReadSectors (Nvme, Buffer, Lba, (UINT32) Blocks);
2101       Blocks = 0;
2102     }
2103
2104     if (EFI_ERROR(Status)) {
2105       DEBUG ((DEBUG_ERROR, "NvmeRead fail, Status = %r\n", Status));
2106       break;
2107     }
2108   }
2109
2110   return Status;
2111 }
2112
2113 /**
2114   Write some blocks to the device.
2115
2116   @param[in] Nvme                   - The pointer to the NVME_CONTEXT Data structure.
2117   @param[in] Buffer                 - The Buffer to be written into the device.
2118   @param[in] Lba                    - The start block number.
2119   @param[in] Blocks                 - Total block number to be written.
2120
2121   @retval EFI_SUCCESS               - Datum are written into the Buffer.
2122   @retval Others                    - Fail to write all the datum.
2123
2124 **/
2125 EFI_STATUS
2126 NvmeWrite (
2127   IN NVME_CONTEXT                  *Nvme,
2128   IN UINT64                        Buffer,
2129   IN UINT64                        Lba,
2130   IN UINTN                         Blocks
2131   )
2132 {
2133   EFI_STATUS                       Status;
2134   UINT32                           BlockSize;
2135   UINT32                           MaxTransferBlocks;
2136
2137   ASSERT (Blocks <= NVME_MAX_SECTORS);
2138   Status        = EFI_SUCCESS;
2139   BlockSize     = Nvme->BlockSize;
2140
2141   if (Nvme->ControllerData->Mdts != 0) {
2142     MaxTransferBlocks = (1 << (Nvme->ControllerData->Mdts)) * (1 << (Nvme->Cap.Mpsmin + 12)) / BlockSize;
2143   } else {
2144     MaxTransferBlocks = 1024;
2145   }
2146
2147   while (Blocks > 0) {
2148     if (Blocks > MaxTransferBlocks) {
2149       Status = NvmeWriteSectors (Nvme, Buffer, Lba, MaxTransferBlocks);
2150
2151       Blocks -= MaxTransferBlocks;
2152       Buffer += (MaxTransferBlocks * BlockSize);
2153       Lba    += MaxTransferBlocks;
2154     } else {
2155       Status = NvmeWriteSectors (Nvme, Buffer, Lba, (UINT32) Blocks);
2156       Blocks = 0;
2157     }
2158
2159     if (EFI_ERROR(Status)) {
2160       DEBUG ((DEBUG_ERROR, "NvmeWrite fail, Status = %r\n", Status));
2161       break;
2162     }
2163   }
2164
2165   return Status;
2166 }