MdeModulePkg/Bus/Pci/NvmExpressPei/NvmExpressPeiHci.c

   1 /** @file
   2   The NvmExpressPei driver is used to manage non-volatile memory subsystem
   3   which follows NVM Express specification at PEI phase.
   4
   5   Copyright (c) 2018 - 2019, Intel Corporation. All rights reserved.<BR>
   6
   7   This program and the accompanying materials
   8   are licensed and made available under the terms and conditions
   9   of the BSD License which accompanies this distribution.  The
  10   full text of the license may be found at
  11   http://opensource.org/licenses/bsd-license.php
  12
  13   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  14   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  15
  16 **/
  17
  18 #include "NvmExpressPei.h"
  19
  20 /**
  21   Transfer MMIO Data to memory.
  22
  23   @param[in,out] MemBuffer    Destination: Memory address.
  24   @param[in] MmioAddr         Source: MMIO address.
  25   @param[in] Size             Size for read.
  26
  27   @retval EFI_SUCCESS         MMIO read sucessfully.
  28
  29 **/
  30 EFI_STATUS
  31 NvmeMmioRead (
  32   IN OUT VOID *MemBuffer,
  33   IN     UINTN MmioAddr,
  34   IN     UINTN Size
  35   )
  36 {
  37   UINTN    Offset;
  38   UINT8    Data;
  39   UINT8    *Ptr;
  40
  41   // priority has adjusted
  42   switch (Size) {
  43     case 4:
  44       *((UINT32 *)MemBuffer) = MmioRead32 (MmioAddr);
  45       break;
  46
  47     case 8:
  48       *((UINT64 *)MemBuffer) = MmioRead64 (MmioAddr);
  49       break;
  50
  51     case 2:
  52       *((UINT16 *)MemBuffer) = MmioRead16 (MmioAddr);
  53       break;
  54
  55     case 1:
  56       *((UINT8 *)MemBuffer) = MmioRead8 (MmioAddr);
  57       break;
  58
  59     default:
  60       Ptr = (UINT8 *)MemBuffer;
  61       for (Offset = 0; Offset < Size; Offset += 1) {
  62         Data = MmioRead8 (MmioAddr + Offset);
  63         Ptr[Offset] = Data;
  64       }
  65       break;
  66   }
  67
  68   return EFI_SUCCESS;
  69 }
  70
  71 /**
  72   Transfer memory data to MMIO.
  73
  74   @param[in,out] MmioAddr    Destination: MMIO address.
  75   @param[in] MemBuffer       Source: Memory address.
  76   @param[in] Size            Size for write.
  77
  78   @retval EFI_SUCCESS        MMIO write sucessfully.
  79
  80 **/
  81 EFI_STATUS
  82 NvmeMmioWrite (
  83   IN OUT UINTN MmioAddr,
  84   IN     VOID *MemBuffer,
  85   IN     UINTN Size
  86   )
  87 {
  88   UINTN    Offset;
  89   UINT8    Data;
  90   UINT8    *Ptr;
  91
  92   // priority has adjusted
  93   switch (Size) {
  94     case 4:
  95       MmioWrite32 (MmioAddr, *((UINT32 *)MemBuffer));
  96       break;
  97
  98     case 8:
  99       MmioWrite64 (MmioAddr, *((UINT64 *)MemBuffer));
 100       break;
 101
 102     case 2:
 103       MmioWrite16 (MmioAddr, *((UINT16 *)MemBuffer));
 104       break;
 105
 106     case 1:
 107       MmioWrite8 (MmioAddr, *((UINT8 *)MemBuffer));
 108       break;
 109
 110     default:
 111       Ptr = (UINT8 *)MemBuffer;
 112       for (Offset = 0; Offset < Size; Offset += 1) {
 113         Data = Ptr[Offset];
 114         MmioWrite8 (MmioAddr + Offset, Data);
 115       }
 116       break;
 117   }
 118
 119   return EFI_SUCCESS;
 120 }
 121
 122 /**
 123   Get the page offset for specific NVME based memory.
 124
 125   @param[in] BaseMemIndex    The Index of BaseMem (0-based).
 126
 127   @retval - The page count for specific BaseMem Index
 128
 129 **/
 130 UINT32
 131 NvmeBaseMemPageOffset (
 132   IN UINTN              BaseMemIndex
 133   )
 134 {
 135   UINT32                Pages;
 136   UINTN                 Index;
 137   UINT32                PageSizeList[5];
 138
 139   PageSizeList[0] = 1;  /* ASQ */
 140   PageSizeList[1] = 1;  /* ACQ */
 141   PageSizeList[2] = 1;  /* SQs */
 142   PageSizeList[3] = 1;  /* CQs */
 143   PageSizeList[4] = NVME_PRP_SIZE;  /* PRPs */
 144
 145   if (BaseMemIndex > MAX_BASEMEM_COUNT) {
 146     DEBUG ((DEBUG_ERROR, "%a: The input BaseMem index is invalid.\n", __FUNCTION__));
 147     ASSERT (FALSE);
 148     return 0;
 149   }
 150
 151   Pages = 0;
 152   for (Index = 0; Index < BaseMemIndex; Index++) {
 153     Pages += PageSizeList[Index];
 154   }
 155
 156   return Pages;
 157 }
 158
 159 /**
 160   Wait for NVME controller status to be ready or not.
 161
 162   @param[in] Private      The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 163   @param[in] WaitReady    Flag for waitting status ready or not.
 164
 165   @return EFI_SUCCESS     Successfully to wait specific status.
 166   @return others          Fail to wait for specific controller status.
 167
 168 **/
 169 EFI_STATUS
 170 NvmeWaitController (
 171   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private,
 172   IN BOOLEAN                             WaitReady
 173   )
 174 {
 175   NVME_CSTS              Csts;
 176   EFI_STATUS             Status;
 177   UINT32                 Index;
 178   UINT8                  Timeout;
 179
 180   //
 181   // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
 182   // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
 183   //
 184   if (Private->Cap.To == 0) {
 185     Timeout = 1;
 186   } else {
 187     Timeout = Private->Cap.To;
 188   }
 189
 190   Status = EFI_SUCCESS;
 191   for(Index = (Timeout * 500); Index != 0; --Index) {
 192     MicroSecondDelay (1000);
 193
 194     //
 195     // Check if the controller is initialized
 196     //
 197     Status = NVME_GET_CSTS (Private, &Csts);
 198     if (EFI_ERROR(Status)) {
 199       DEBUG ((DEBUG_ERROR, "%a: NVME_GET_CSTS fail, Status - %r\n", __FUNCTION__, Status));
 200       return Status;
 201     }
 202
 203     if ((BOOLEAN) Csts.Rdy == WaitReady) {
 204       break;
 205     }
 206   }
 207
 208   if (Index == 0) {
 209     Status = EFI_TIMEOUT;
 210   }
 211
 212   return Status;
 213 }
 214
 215 /**
 216   Disable the Nvm Express controller.
 217
 218   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 219
 220   @return EFI_SUCCESS    Successfully disable the controller.
 221   @return others         Fail to disable the controller.
 222
 223 **/
 224 EFI_STATUS
 225 NvmeDisableController (
 226   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 227   )
 228 {
 229   NVME_CC       Cc;
 230   NVME_CSTS     Csts;
 231   EFI_STATUS    Status;
 232
 233   Status = NVME_GET_CSTS (Private, &Csts);
 234
 235   //
 236   // Read Controller Configuration Register.
 237   //
 238   Status = NVME_GET_CC (Private, &Cc);
 239   if (EFI_ERROR (Status)) {
 240     DEBUG ((DEBUG_ERROR, "%a: NVME_GET_CC fail, Status - %r\n", __FUNCTION__, Status));
 241     goto ErrorExit;
 242   }
 243
 244   if (Cc.En == 1) {
 245     Cc.En = 0;
 246     //
 247     // Disable the controller.
 248     //
 249     Status = NVME_SET_CC (Private, &Cc);
 250     if (EFI_ERROR (Status)) {
 251       DEBUG ((DEBUG_ERROR, "%a: NVME_SET_CC fail, Status - %r\n", __FUNCTION__, Status));
 252       goto ErrorExit;
 253     }
 254   }
 255
 256   Status = NvmeWaitController (Private, FALSE);
 257   if (EFI_ERROR (Status)) {
 258     DEBUG ((DEBUG_ERROR, "%a: NvmeWaitController fail, Status - %r\n", __FUNCTION__, Status));
 259     goto ErrorExit;
 260   }
 261
 262   return EFI_SUCCESS;
 263
 264 ErrorExit:
 265   DEBUG ((DEBUG_ERROR, "%a fail, Status - %r\n", __FUNCTION__, Status));
 266   return Status;
 267 }
 268
 269 /**
 270   Enable the Nvm Express controller.
 271
 272   @param[in] Private    The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 273
 274   @return EFI_SUCCESS         Successfully enable the controller.
 275   @return EFI_DEVICE_ERROR    Fail to enable the controller.
 276   @return EFI_TIMEOUT         Fail to enable the controller in given time slot.
 277
 278 **/
 279 EFI_STATUS
 280 NvmeEnableController (
 281   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 282   )
 283 {
 284   NVME_CC       Cc;
 285   EFI_STATUS    Status;
 286
 287   //
 288   // Enable the controller
 289   // CC.AMS, CC.MPS and CC.CSS are all set to 0
 290   //
 291   ZeroMem (&Cc, sizeof (NVME_CC));
 292   Cc.En     = 1;
 293   Cc.Iosqes = 6;
 294   Cc.Iocqes = 4;
 295   Status    = NVME_SET_CC (Private, &Cc);
 296   if (EFI_ERROR (Status)) {
 297     DEBUG ((DEBUG_ERROR, "%a: NVME_SET_CC fail, Status - %r\n", __FUNCTION__, Status));
 298     goto ErrorExit;
 299   }
 300
 301   Status = NvmeWaitController (Private, TRUE);
 302   if (EFI_ERROR (Status)) {
 303     DEBUG ((DEBUG_ERROR, "%a: NvmeWaitController fail, Status - %r\n", __FUNCTION__, Status));
 304     goto ErrorExit;
 305   }
 306
 307   return EFI_SUCCESS;
 308
 309 ErrorExit:
 310   DEBUG ((DEBUG_ERROR, "%a fail, Status: %r\n", __FUNCTION__, Status));
 311   return Status;
 312 }
 313
 314 /**
 315   Get the Identify Controller data.
 316
 317   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 318   @param[in] Buffer      The Buffer used to store the Identify Controller data.
 319
 320   @return EFI_SUCCESS    Successfully get the Identify Controller data.
 321   @return others         Fail to get the Identify Controller data.
 322
 323 **/
 324 EFI_STATUS
 325 NvmeIdentifyController (
 326   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private,
 327   IN VOID                                *Buffer
 328   )
 329 {
 330   EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET    CommandPacket;
 331   EDKII_PEI_NVM_EXPRESS_COMMAND                     Command;
 332   EDKII_PEI_NVM_EXPRESS_COMPLETION                  Completion;
 333   EFI_STATUS                                        Status;
 334
 335   ZeroMem (&CommandPacket, sizeof(EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 336   ZeroMem (&Command, sizeof(EDKII_PEI_NVM_EXPRESS_COMMAND));
 337   ZeroMem (&Completion, sizeof(EDKII_PEI_NVM_EXPRESS_COMPLETION));
 338
 339   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;
 340   //
 341   // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
 342   // For the Identify command, the Namespace Identifier is only used for the Namespace Data structure.
 343   //
 344   Command.Nsid        = 0;
 345
 346   CommandPacket.NvmeCmd        = &Command;
 347   CommandPacket.NvmeCompletion = &Completion;
 348   CommandPacket.TransferBuffer = Buffer;
 349   CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA);
 350   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 351   CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 352   //
 353   // Set bit 0 (Cns bit) to 1 to identify the controller
 354   //
 355   CommandPacket.NvmeCmd->Cdw10 = 1;
 356   CommandPacket.NvmeCmd->Flags = CDW10_VALID;
 357
 358   Status = NvmePassThru (
 359              Private,
 360              NVME_CONTROLLER_NSID,
 361              &CommandPacket
 362              );
 363   return Status;
 364 }
 365
 366 /**
 367   Get specified identify namespace data.
 368
 369   @param[in] Private        The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 370   @param[in] NamespaceId    The specified namespace identifier.
 371   @param[in] Buffer         The buffer used to store the identify namespace data.
 372
 373   @return EFI_SUCCESS         Successfully get the identify namespace data.
 374   @return EFI_DEVICE_ERROR    Fail to get the identify namespace data.
 375
 376 **/
 377 EFI_STATUS
 378 NvmeIdentifyNamespace (
 379   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private,
 380   IN UINT32                              NamespaceId,
 381   IN VOID                                *Buffer
 382   )
 383 {
 384   EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET    CommandPacket;
 385   EDKII_PEI_NVM_EXPRESS_COMMAND                     Command;
 386   EDKII_PEI_NVM_EXPRESS_COMPLETION                  Completion;
 387   EFI_STATUS                                        Status;
 388
 389   ZeroMem (&CommandPacket, sizeof(EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 390   ZeroMem (&Command, sizeof(EDKII_PEI_NVM_EXPRESS_COMMAND));
 391   ZeroMem (&Completion, sizeof(EDKII_PEI_NVM_EXPRESS_COMPLETION));
 392
 393   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;
 394   Command.Nsid        = NamespaceId;
 395
 396   CommandPacket.NvmeCmd        = &Command;
 397   CommandPacket.NvmeCompletion = &Completion;
 398   CommandPacket.TransferBuffer = Buffer;
 399   CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA);
 400   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 401   CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 402   //
 403   // Set bit 0 (Cns bit) to 1 to identify a namespace
 404   //
 405   CommandPacket.NvmeCmd->Cdw10 = 0;
 406   CommandPacket.NvmeCmd->Flags = CDW10_VALID;
 407
 408   Status = NvmePassThru (
 409              Private,
 410              NamespaceId,
 411              &CommandPacket
 412              );
 413   return Status;
 414 }
 415
 416 /**
 417   Dump the Identify Controller data.
 418
 419   @param[in] ControllerData    The pointer to the NVME_ADMIN_CONTROLLER_DATA data structure.
 420
 421 **/
 422 VOID
 423 NvmeDumpControllerData (
 424   IN NVME_ADMIN_CONTROLLER_DATA    *ControllerData
 425   )
 426 {
 427   UINT8    Sn[21];
 428   UINT8    Mn[41];
 429
 430   CopyMem (Sn, ControllerData->Sn, sizeof (ControllerData->Sn));
 431   Sn[20] = 0;
 432   CopyMem (Mn, ControllerData->Mn, sizeof (ControllerData->Mn));
 433   Mn[40] = 0;
 434
 435   DEBUG ((DEBUG_INFO, " == NVME IDENTIFY CONTROLLER DATA ==\n"));
 436   DEBUG ((DEBUG_INFO, "    PCI VID   : 0x%x\n", ControllerData->Vid));
 437   DEBUG ((DEBUG_INFO, "    PCI SSVID : 0x%x\n", ControllerData->Ssvid));
 438   DEBUG ((DEBUG_INFO, "    SN        : %a\n",   Sn));
 439   DEBUG ((DEBUG_INFO, "    MN        : %a\n",   Mn));
 440   DEBUG ((DEBUG_INFO, "    FR        : 0x%lx\n", *((UINT64*)ControllerData->Fr)));
 441   DEBUG ((DEBUG_INFO, "    RAB       : 0x%x\n", ControllerData->Rab));
 442   DEBUG ((DEBUG_INFO, "    IEEE      : 0x%x\n", *(UINT32*)ControllerData->Ieee_oui));
 443   DEBUG ((DEBUG_INFO, "    AERL      : 0x%x\n", ControllerData->Aerl));
 444   DEBUG ((DEBUG_INFO, "    SQES      : 0x%x\n", ControllerData->Sqes));
 445   DEBUG ((DEBUG_INFO, "    CQES      : 0x%x\n", ControllerData->Cqes));
 446   DEBUG ((DEBUG_INFO, "    NN        : 0x%x\n", ControllerData->Nn));
 447   return;
 448 }
 449
 450 /**
 451   Create IO completion queue.
 452
 453   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 454
 455   @return EFI_SUCCESS    Successfully create io completion queue.
 456   @return others         Fail to create io completion queue.
 457
 458 **/
 459 EFI_STATUS
 460 NvmeCreateIoCompletionQueue (
 461   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 462   )
 463 {
 464   EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET    CommandPacket;
 465   EDKII_PEI_NVM_EXPRESS_COMMAND                     Command;
 466   EDKII_PEI_NVM_EXPRESS_COMPLETION                  Completion;
 467   EFI_STATUS                                        Status;
 468   NVME_ADMIN_CRIOCQ                                 CrIoCq;
 469
 470   ZeroMem (&CommandPacket, sizeof(EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 471   ZeroMem (&Command, sizeof(EDKII_PEI_NVM_EXPRESS_COMMAND));
 472   ZeroMem (&Completion, sizeof(EDKII_PEI_NVM_EXPRESS_COMPLETION));
 473   ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ));
 474
 475   CommandPacket.NvmeCmd        = &Command;
 476   CommandPacket.NvmeCompletion = &Completion;
 477
 478   Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_CMD;
 479   Command.Cdw0.Cid    = Private->Cid[NVME_ADMIN_QUEUE]++;
 480   CommandPacket.TransferBuffer = Private->CqBuffer[NVME_IO_QUEUE];
 481   CommandPacket.TransferLength = EFI_PAGE_SIZE;
 482   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 483   CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 484
 485   CrIoCq.Qid   = NVME_IO_QUEUE;
 486   CrIoCq.Qsize = NVME_CCQ_SIZE;
 487   CrIoCq.Pc    = 1;
 488   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
 489   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
 490
 491   Status = NvmePassThru (
 492              Private,
 493              NVME_CONTROLLER_NSID,
 494              &CommandPacket
 495              );
 496   return Status;
 497 }
 498
 499 /**
 500   Create IO submission queue.
 501
 502   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 503
 504   @return EFI_SUCCESS    Successfully create io submission queue.
 505   @return others         Fail to create io submission queue.
 506
 507 **/
 508 EFI_STATUS
 509 NvmeCreateIoSubmissionQueue (
 510   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 511   )
 512 {
 513   EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET    CommandPacket;
 514   EDKII_PEI_NVM_EXPRESS_COMMAND                     Command;
 515   EDKII_PEI_NVM_EXPRESS_COMPLETION                  Completion;
 516   EFI_STATUS                                        Status;
 517   NVME_ADMIN_CRIOSQ                                 CrIoSq;
 518
 519   ZeroMem (&CommandPacket, sizeof(EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 520   ZeroMem (&Command, sizeof(EDKII_PEI_NVM_EXPRESS_COMMAND));
 521   ZeroMem (&Completion, sizeof(EDKII_PEI_NVM_EXPRESS_COMPLETION));
 522   ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ));
 523
 524   CommandPacket.NvmeCmd        = &Command;
 525   CommandPacket.NvmeCompletion = &Completion;
 526
 527   Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_CMD;
 528   Command.Cdw0.Cid    = Private->Cid[NVME_ADMIN_QUEUE]++;
 529   CommandPacket.TransferBuffer = Private->SqBuffer[NVME_IO_QUEUE];
 530   CommandPacket.TransferLength = EFI_PAGE_SIZE;
 531   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 532   CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 533
 534   CrIoSq.Qid   = NVME_IO_QUEUE;
 535   CrIoSq.Qsize = NVME_CSQ_SIZE;
 536   CrIoSq.Pc    = 1;
 537   CrIoSq.Cqid  = NVME_IO_QUEUE;
 538   CrIoSq.Qprio = 0;
 539   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
 540   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
 541
 542   Status = NvmePassThru (
 543              Private,
 544              NVME_CONTROLLER_NSID,
 545              &CommandPacket
 546              );
 547   return Status;
 548 }
 549
 550 /**
 551   Initialize the Nvm Express controller.
 552
 553   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 554
 555   @retval EFI_SUCCESS    The NVM Express Controller is initialized successfully.
 556   @retval Others         A device error occurred while initializing the controller.
 557
 558 **/
 559 EFI_STATUS
 560 NvmeControllerInit (
 561   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 562   )
 563 {
 564   EFI_STATUS    Status;
 565   UINTN         Index;
 566   NVME_AQA      Aqa;
 567   NVME_ASQ      Asq;
 568   NVME_ACQ      Acq;
 569   NVME_VER      Ver;
 570
 571   //
 572   // Dump the NVME controller implementation version
 573   //
 574   NVME_GET_VER (Private, &Ver);
 575   DEBUG ((DEBUG_INFO, "NVME controller implementation version: %d.%d\n", Ver.Mjr, Ver.Mnr));
 576
 577   //
 578   // Read the controller Capabilities register and verify that the NVM command set is supported
 579   //
 580   NVME_GET_CAP (Private, &Private->Cap);
 581   if (Private->Cap.Css != 0x01) {
 582     DEBUG ((DEBUG_ERROR, "%a: The NVME controller doesn't support NVMe command set.\n", __FUNCTION__));
 583     return EFI_UNSUPPORTED;
 584   }
 585
 586   //
 587   // Currently, the driver only supports 4k page size
 588   //
 589   if ((Private->Cap.Mpsmin + 12) > EFI_PAGE_SHIFT) {
 590     DEBUG ((DEBUG_ERROR, "%a: The driver doesn't support page size other than 4K.\n", __FUNCTION__));
 591     ASSERT (FALSE);
 592     return EFI_UNSUPPORTED;
 593   }
 594
 595   for (Index = 0; Index < NVME_MAX_QUEUES; Index++) {
 596     Private->Pt[Index]  = 0;
 597     Private->Cid[Index] = 0;
 598     ZeroMem ((VOID *)(UINTN)(&Private->SqTdbl[Index]), sizeof (NVME_SQTDBL));
 599     ZeroMem ((VOID *)(UINTN)(&Private->CqHdbl[Index]), sizeof (NVME_CQHDBL));
 600   }
 601   ZeroMem (Private->Buffer, EFI_PAGE_SIZE * NVME_MEM_MAX_PAGES);
 602
 603   //
 604   // Disable the NVME controller first
 605   //
 606   Status = NvmeDisableController (Private);
 607   if (EFI_ERROR (Status)) {
 608     DEBUG ((DEBUG_ERROR, "%a: NvmeDisableController fail, Status - %r\n", __FUNCTION__, Status));
 609     return Status;
 610   }
 611
 612   //
 613   // Set the number of entries in admin submission & completion queues
 614   //
 615   Aqa.Asqs  = NVME_ASQ_SIZE;
 616   Aqa.Rsvd1 = 0;
 617   Aqa.Acqs  = NVME_ACQ_SIZE;
 618   Aqa.Rsvd2 = 0;
 619
 620   //
 621   // Address of admin submission & completion queues
 622   //
 623   Asq = (UINT64)(UINTN)(NVME_ASQ_BASE (Private) & ~0xFFF);
 624   Acq = (UINT64)(UINTN)(NVME_ACQ_BASE (Private) & ~0xFFF);
 625
 626   //
 627   // Address of I/O submission & completion queues
 628   //
 629   Private->SqBuffer[0] = (NVME_SQ *)(UINTN)NVME_ASQ_BASE (Private);    // NVME_ADMIN_QUEUE
 630   Private->CqBuffer[0] = (NVME_CQ *)(UINTN)NVME_ACQ_BASE (Private);    // NVME_ADMIN_QUEUE
 631   Private->SqBuffer[1] = (NVME_SQ *)(UINTN)NVME_SQ_BASE (Private, 0);  // NVME_IO_QUEUE
 632   Private->CqBuffer[1] = (NVME_CQ *)(UINTN)NVME_CQ_BASE (Private, 0);  // NVME_IO_QUEUE
 633   DEBUG ((DEBUG_INFO, "Admin Submission Queue Size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));
 634   DEBUG ((DEBUG_INFO, "Admin Completion Queue Size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));
 635   DEBUG ((DEBUG_INFO, "Admin Submission Queue (SqBuffer[0]) =   [%08X]\n", Private->SqBuffer[0]));
 636   DEBUG ((DEBUG_INFO, "Admin Completion Queue (CqBuffer[0]) =   [%08X]\n", Private->CqBuffer[0]));
 637   DEBUG ((DEBUG_INFO, "I/O   Submission Queue (SqBuffer[1]) =   [%08X]\n", Private->SqBuffer[1]));
 638   DEBUG ((DEBUG_INFO, "I/O   Completion Queue (CqBuffer[1]) =   [%08X]\n", Private->CqBuffer[1]));
 639
 640   //
 641   // Program admin queue attributes
 642   //
 643   NVME_SET_AQA (Private, &Aqa);
 644
 645   //
 646   // Program admin submission & completion queues address
 647   //
 648   NVME_SET_ASQ (Private, &Asq);
 649   NVME_SET_ACQ (Private, &Acq);
 650
 651   //
 652   // Enable the NVME controller
 653   //
 654   Status = NvmeEnableController (Private);
 655   if (EFI_ERROR (Status)) {
 656     DEBUG ((DEBUG_ERROR, "%a: NvmeEnableController fail, Status - %r\n", __FUNCTION__, Status));
 657     return Status;
 658   }
 659
 660   //
 661   // Get the Identify Controller data
 662   //
 663   if (Private->ControllerData == NULL) {
 664     Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_CONTROLLER_DATA));
 665     if (Private->ControllerData == NULL) {
 666       return EFI_OUT_OF_RESOURCES;
 667     }
 668   }
 669   Status = NvmeIdentifyController (Private, Private->ControllerData);
 670   if (EFI_ERROR (Status)) {
 671     DEBUG ((DEBUG_ERROR, "%a: NvmeIdentifyController fail, Status - %r\n", __FUNCTION__, Status));
 672     return Status;
 673   }
 674   NvmeDumpControllerData (Private->ControllerData);
 675
 676   //
 677   // Check the namespace number for storing the namespaces information
 678   //
 679   if (Private->ControllerData->Nn > MAX_UINT32 / sizeof (PEI_NVME_NAMESPACE_INFO)) {
 680     DEBUG ((
 681       DEBUG_ERROR,
 682       "%a: Number of Namespaces field in Identify Controller data not supported by the driver.\n",
 683       __FUNCTION__
 684       ));
 685     return EFI_UNSUPPORTED;
 686   }
 687
 688   //
 689   // Create one I/O completion queue and one I/O submission queue
 690   //
 691   Status = NvmeCreateIoCompletionQueue (Private);
 692   if (EFI_ERROR (Status)) {
 693     DEBUG ((DEBUG_ERROR, "%a: Create IO completion queue fail, Status - %r\n", __FUNCTION__, Status));
 694     return Status;
 695   }
 696   Status = NvmeCreateIoSubmissionQueue (Private);
 697   if (EFI_ERROR (Status)) {
 698     DEBUG ((DEBUG_ERROR, "%a: Create IO submission queue fail, Status - %r\n", __FUNCTION__, Status));
 699   }
 700
 701   return Status;
 702 }
 703
 704 /**
 705   Free the DMA resources allocated by an NVME controller.
 706
 707   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 708
 709 **/
 710 VOID
 711 NvmeFreeDmaResource (
 712   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 713   )
 714 {
 715   ASSERT (Private != NULL);
 716
 717   if (Private->BufferMapping != NULL) {
 718     IoMmuFreeBuffer (
 719        NVME_MEM_MAX_PAGES,
 720        Private->Buffer,
 721        Private->BufferMapping
 722        );
 723   }
 724
 725   return;
 726 }