MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c

   1 /** @file
   2   NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
   3   NVM Express specification.
   4
   5   Copyright (c) 2013, Intel Corporation. All rights reserved.<BR>
   6   This program and the accompanying materials
   7   are licensed and made available under the terms and conditions of the BSD License
   8   which accompanies this distribution.  The full text of the license may be found at
   9   http://opensource.org/licenses/bsd-license.php.
  10
  11   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13
  14 **/
  15
  16 #include "NvmExpress.h"
  17
  18 /**
  19   Read Nvm Express controller capability register.
  20
  21   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
  22   @param  Cap              The buffer used to store capability register content.
  23
  24   @return EFI_SUCCESS      Successfully read the controller capability register content.
  25   @return EFI_DEVICE_ERROR Fail to read the controller capability register.
  26
  27 **/
  28 EFI_STATUS
  29 ReadNvmeControllerCapabilities (
  30   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
  31   IN NVME_CAP                         *Cap
  32   )
  33 {
  34   EFI_PCI_IO_PROTOCOL   *PciIo;
  35   EFI_STATUS            Status;
  36   UINT64                Data;
  37
  38   PciIo  = Private->PciIo;
  39   Status = PciIo->Mem.Read (
  40                         PciIo,
  41                         EfiPciIoWidthUint32,
  42                         NVME_BAR,
  43                         NVME_CAP_OFFSET,
  44                         2,
  45                         &Data
  46                         );
  47
  48   if (EFI_ERROR(Status)) {
  49     return Status;
  50   }
  51
  52   WriteUnaligned64 ((UINT64*)Cap, Data);
  53   return EFI_SUCCESS;
  54 }
  55
  56 /**
  57   Read Nvm Express controller configuration register.
  58
  59   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
  60   @param  Cc               The buffer used to store configuration register content.
  61
  62   @return EFI_SUCCESS      Successfully read the controller configuration register content.
  63   @return EFI_DEVICE_ERROR Fail to read the controller configuration register.
  64
  65 **/
  66 EFI_STATUS
  67 ReadNvmeControllerConfiguration (
  68   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
  69   IN NVME_CC                          *Cc
  70   )
  71 {
  72   EFI_PCI_IO_PROTOCOL   *PciIo;
  73   EFI_STATUS            Status;
  74   UINT32                Data;
  75
  76   PciIo  = Private->PciIo;
  77   Status = PciIo->Mem.Read (
  78                         PciIo,
  79                         EfiPciIoWidthUint32,
  80                         NVME_BAR,
  81                         NVME_CC_OFFSET,
  82                         1,
  83                         &Data
  84                         );
  85
  86   if (EFI_ERROR(Status)) {
  87     return Status;
  88   }
  89
  90   WriteUnaligned32 ((UINT32*)Cc, Data);
  91   return EFI_SUCCESS;
  92 }
  93
  94 /**
  95   Write Nvm Express controller configuration register.
  96
  97   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
  98   @param  Cc               The buffer used to store the content to be written into configuration register.
  99
 100   @return EFI_SUCCESS      Successfully write data into the controller configuration register.
 101   @return EFI_DEVICE_ERROR Fail to write data into the controller configuration register.
 102
 103 **/
 104 EFI_STATUS
 105 WriteNvmeControllerConfiguration (
 106   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
 107   IN NVME_CC                          *Cc
 108   )
 109 {
 110   EFI_PCI_IO_PROTOCOL   *PciIo;
 111   EFI_STATUS            Status;
 112   UINT32                Data;
 113
 114   PciIo  = Private->PciIo;
 115   Data   = ReadUnaligned32 ((UINT32*)Cc);
 116   Status = PciIo->Mem.Write (
 117                         PciIo,
 118                         EfiPciIoWidthUint32,
 119                         NVME_BAR,
 120                         NVME_CC_OFFSET,
 121                         1,
 122                         &Data
 123                         );
 124
 125   if (EFI_ERROR(Status)) {
 126     return Status;
 127   }
 128
 129   DEBUG ((EFI_D_INFO, "Cc.En: %d\n", Cc->En));
 130   DEBUG ((EFI_D_INFO, "Cc.Css: %d\n", Cc->Css));
 131   DEBUG ((EFI_D_INFO, "Cc.Mps: %d\n", Cc->Mps));
 132   DEBUG ((EFI_D_INFO, "Cc.Ams: %d\n", Cc->Ams));
 133   DEBUG ((EFI_D_INFO, "Cc.Shn: %d\n", Cc->Shn));
 134   DEBUG ((EFI_D_INFO, "Cc.Iosqes: %d\n", Cc->Iosqes));
 135   DEBUG ((EFI_D_INFO, "Cc.Iocqes: %d\n", Cc->Iocqes));
 136
 137   return EFI_SUCCESS;
 138 }
 139
 140 /**
 141   Read Nvm Express controller status register.
 142
 143   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 144   @param  Csts             The buffer used to store status register content.
 145
 146   @return EFI_SUCCESS      Successfully read the controller status register content.
 147   @return EFI_DEVICE_ERROR Fail to read the controller status register.
 148
 149 **/
 150 EFI_STATUS
 151 ReadNvmeControllerStatus (
 152   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
 153   IN NVME_CSTS                        *Csts
 154   )
 155 {
 156   EFI_PCI_IO_PROTOCOL   *PciIo;
 157   EFI_STATUS            Status;
 158   UINT32                Data;
 159
 160   PciIo  = Private->PciIo;
 161   Status = PciIo->Mem.Read (
 162                         PciIo,
 163                         EfiPciIoWidthUint32,
 164                         NVME_BAR,
 165                         NVME_CSTS_OFFSET,
 166                         1,
 167                         &Data
 168                         );
 169
 170   if (EFI_ERROR(Status)) {
 171     return Status;
 172   }
 173
 174   WriteUnaligned32 ((UINT32*)Csts, Data);
 175   return EFI_SUCCESS;
 176 }
 177
 178 /**
 179   Read Nvm Express admin queue attributes register.
 180
 181   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 182   @param  Aqa              The buffer used to store admin queue attributes register content.
 183
 184   @return EFI_SUCCESS      Successfully read the admin queue attributes register content.
 185   @return EFI_DEVICE_ERROR Fail to read the admin queue attributes register.
 186
 187 **/
 188 EFI_STATUS
 189 ReadNvmeAdminQueueAttributes (
 190   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
 191   IN NVME_AQA                         *Aqa
 192   )
 193 {
 194   EFI_PCI_IO_PROTOCOL   *PciIo;
 195   EFI_STATUS            Status;
 196   UINT32                Data;
 197
 198   PciIo  = Private->PciIo;
 199   Status = PciIo->Mem.Read (
 200                         PciIo,
 201                         EfiPciIoWidthUint32,
 202                         NVME_BAR,
 203                         NVME_AQA_OFFSET,
 204                         1,
 205                         &Data
 206                         );
 207
 208   if (EFI_ERROR(Status)) {
 209     return Status;
 210   }
 211
 212   WriteUnaligned32 ((UINT32*)Aqa, Data);
 213   return EFI_SUCCESS;
 214 }
 215
 216 /**
 217   Write Nvm Express admin queue attributes register.
 218
 219   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 220   @param  Aqa              The buffer used to store the content to be written into admin queue attributes register.
 221
 222   @return EFI_SUCCESS      Successfully write data into the admin queue attributes register.
 223   @return EFI_DEVICE_ERROR Fail to write data into the admin queue attributes register.
 224
 225 **/
 226 EFI_STATUS
 227 WriteNvmeAdminQueueAttributes (
 228   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
 229   IN NVME_AQA                         *Aqa
 230   )
 231 {
 232   EFI_PCI_IO_PROTOCOL   *PciIo;
 233   EFI_STATUS            Status;
 234   UINT32                Data;
 235
 236   PciIo  = Private->PciIo;
 237   Data   = ReadUnaligned32 ((UINT32*)Aqa);
 238   Status = PciIo->Mem.Write (
 239                         PciIo,
 240                         EfiPciIoWidthUint32,
 241                         NVME_BAR,
 242                         NVME_AQA_OFFSET,
 243                         1,
 244                         &Data
 245                         );
 246
 247   if (EFI_ERROR(Status)) {
 248     return Status;
 249   }
 250
 251   DEBUG ((EFI_D_INFO, "Aqa.Asqs: %d\n", Aqa->Asqs));
 252   DEBUG ((EFI_D_INFO, "Aqa.Acqs: %d\n", Aqa->Acqs));
 253
 254   return EFI_SUCCESS;
 255 }
 256
 257 /**
 258   Read Nvm Express admin submission queue base address register.
 259
 260   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 261   @param  Asq              The buffer used to store admin submission queue base address register content.
 262
 263   @return EFI_SUCCESS      Successfully read the admin submission queue base address register content.
 264   @return EFI_DEVICE_ERROR Fail to read the admin submission queue base address register.
 265
 266 **/
 267 EFI_STATUS
 268 ReadNvmeAdminSubmissionQueueBaseAddress (
 269   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
 270   IN NVME_ASQ                         *Asq
 271   )
 272 {
 273   EFI_PCI_IO_PROTOCOL   *PciIo;
 274   EFI_STATUS            Status;
 275   UINT64                Data;
 276
 277   PciIo  = Private->PciIo;
 278   Status = PciIo->Mem.Read (
 279                         PciIo,
 280                         EfiPciIoWidthUint32,
 281                         NVME_BAR,
 282                         NVME_ASQ_OFFSET,
 283                         2,
 284                         &Data
 285                         );
 286
 287   if (EFI_ERROR(Status)) {
 288     return Status;
 289   }
 290
 291   WriteUnaligned64 ((UINT64*)Asq, Data);
 292   return EFI_SUCCESS;
 293 }
 294
 295 /**
 296   Write Nvm Express admin submission queue base address register.
 297
 298   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 299   @param  Asq              The buffer used to store the content to be written into admin submission queue base address register.
 300
 301   @return EFI_SUCCESS      Successfully write data into the admin submission queue base address register.
 302   @return EFI_DEVICE_ERROR Fail to write data into the admin submission queue base address register.
 303
 304 **/
 305 EFI_STATUS
 306 WriteNvmeAdminSubmissionQueueBaseAddress (
 307   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
 308   IN NVME_ASQ                         *Asq
 309   )
 310 {
 311   EFI_PCI_IO_PROTOCOL   *PciIo;
 312   EFI_STATUS            Status;
 313   UINT64                Data;
 314
 315   PciIo  = Private->PciIo;
 316   Data   = ReadUnaligned64 ((UINT64*)Asq);
 317
 318   Status = PciIo->Mem.Write (
 319                         PciIo,
 320                         EfiPciIoWidthUint32,
 321                         NVME_BAR,
 322                         NVME_ASQ_OFFSET,
 323                         2,
 324                         &Data
 325                         );
 326
 327   if (EFI_ERROR(Status)) {
 328     return Status;
 329   }
 330
 331   DEBUG ((EFI_D_INFO, "Asq.Asqb: %lx\n", Asq->Asqb));
 332
 333   return EFI_SUCCESS;
 334 }
 335
 336 /**
 337   Read Nvm Express admin completion queue base address register.
 338
 339   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 340   @param  Acq              The buffer used to store admin completion queue base address register content.
 341
 342   @return EFI_SUCCESS      Successfully read the admin completion queue base address register content.
 343   @return EFI_DEVICE_ERROR Fail to read the admin completion queue base address register.
 344
 345 **/
 346 EFI_STATUS
 347 ReadNvmeAdminCompletionQueueBaseAddress (
 348   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
 349   IN NVME_ACQ                         *Acq
 350   )
 351 {
 352   EFI_PCI_IO_PROTOCOL   *PciIo;
 353   EFI_STATUS            Status;
 354   UINT64                Data;
 355
 356   PciIo  = Private->PciIo;
 357
 358   Status = PciIo->Mem.Read (
 359                         PciIo,
 360                         EfiPciIoWidthUint32,
 361                         NVME_BAR,
 362                         NVME_ACQ_OFFSET,
 363                         2,
 364                         &Data
 365                         );
 366
 367   if (EFI_ERROR(Status)) {
 368     return Status;
 369   }
 370
 371   WriteUnaligned64 ((UINT64*)Acq, Data);
 372   return EFI_SUCCESS;
 373 }
 374
 375 /**
 376   Write Nvm Express admin completion queue base address register.
 377
 378   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 379   @param  Acq              The buffer used to store the content to be written into admin completion queue base address register.
 380
 381   @return EFI_SUCCESS      Successfully write data into the admin completion queue base address register.
 382   @return EFI_DEVICE_ERROR Fail to write data into the admin completion queue base address register.
 383
 384 **/
 385 EFI_STATUS
 386 WriteNvmeAdminCompletionQueueBaseAddress (
 387   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
 388   IN NVME_ACQ                         *Acq
 389   )
 390 {
 391   EFI_PCI_IO_PROTOCOL   *PciIo;
 392   EFI_STATUS            Status;
 393   UINT64                Data;
 394
 395   PciIo  = Private->PciIo;
 396   Data   = ReadUnaligned64 ((UINT64*)Acq);
 397
 398   Status = PciIo->Mem.Write (
 399                         PciIo,
 400                         EfiPciIoWidthUint32,
 401                         NVME_BAR,
 402                         NVME_ACQ_OFFSET,
 403                         2,
 404                         &Data
 405                         );
 406
 407   if (EFI_ERROR(Status)) {
 408     return Status;
 409   }
 410
 411   DEBUG ((EFI_D_INFO, "Acq.Acqb: %lxh\n", Acq->Acqb));
 412
 413   return EFI_SUCCESS;
 414 }
 415
 416 /**
 417   Disable the Nvm Express controller.
 418
 419   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 420
 421   @return EFI_SUCCESS      Successfully disable the controller.
 422   @return EFI_DEVICE_ERROR Fail to disable the controller.
 423
 424 **/
 425 EFI_STATUS
 426 NvmeDisableController (
 427   IN NVME_CONTROLLER_PRIVATE_DATA     *Private
 428   )
 429 {
 430   NVME_CC                Cc;
 431   NVME_CSTS              Csts;
 432   EFI_STATUS             Status;
 433
 434   //
 435   // Read Controller Configuration Register.
 436   //
 437   Status = ReadNvmeControllerConfiguration (Private, &Cc);
 438   if (EFI_ERROR(Status)) {
 439     return Status;
 440   }
 441
 442   Cc.En = 0;
 443
 444   //
 445   // Disable the controller.
 446   //
 447   Status = WriteNvmeControllerConfiguration (Private, &Cc);
 448
 449   if (EFI_ERROR(Status)) {
 450     return Status;
 451   }
 452
 453   gBS->Stall(10000);
 454
 455   //
 456   // Check if the controller is reset
 457   //
 458   Status = ReadNvmeControllerStatus (Private, &Csts);
 459
 460   if (EFI_ERROR(Status)) {
 461     return Status;
 462   }
 463
 464   if (Csts.Rdy != 0) {
 465     return EFI_DEVICE_ERROR;
 466   }
 467
 468   DEBUG ((EFI_D_INFO, "NVMe controller is disabled with status [%r].\n", Status));
 469   return Status;
 470 }
 471
 472 /**
 473   Enable the Nvm Express controller.
 474
 475   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 476
 477   @return EFI_SUCCESS      Successfully enable the controller.
 478   @return EFI_DEVICE_ERROR Fail to enable the controller.
 479   @return EFI_TIMEOUT      Fail to enable the controller in given time slot.
 480
 481 **/
 482 EFI_STATUS
 483 NvmeEnableController (
 484   IN NVME_CONTROLLER_PRIVATE_DATA     *Private
 485   )
 486 {
 487   NVME_CC                Cc;
 488   NVME_CSTS              Csts;
 489   EFI_STATUS             Status;
 490   UINT32                 Index;
 491   UINT8                  Timeout;
 492
 493   //
 494   // Enable the controller
 495   //
 496   ZeroMem (&Cc, sizeof (NVME_CC));
 497   Cc.En     = 1;
 498   Cc.Iosqes = 6;
 499   Cc.Iocqes = 4;
 500   Status    = WriteNvmeControllerConfiguration (Private, &Cc);
 501
 502   if (EFI_ERROR(Status)) {
 503     return Status;
 504   }
 505
 506   //
 507   // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
 508   // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
 509   //
 510   if (Private->Cap.To == 0) {
 511     Timeout = 1;
 512   } else {
 513     Timeout = Private->Cap.To;
 514   }
 515
 516   for(Index = (Timeout * 500); Index != 0; --Index) {
 517     gBS->Stall(1000);
 518
 519     //
 520     // Check if the controller is initialized
 521     //
 522     Status = ReadNvmeControllerStatus (Private, &Csts);
 523
 524     if (EFI_ERROR(Status)) {
 525       return Status;
 526     }
 527
 528     if (Csts.Rdy) {
 529       break;
 530     }
 531   }
 532
 533   if (Index == 0) {
 534     Status = EFI_TIMEOUT;
 535   }
 536
 537   DEBUG ((EFI_D_INFO, "NVMe controller is enabled with status [%r].\n", Status));
 538   return Status;
 539 }
 540
 541 /**
 542   Get identify controller data.
 543
 544   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 545   @param  Buffer           The buffer used to store the identify controller data.
 546
 547   @return EFI_SUCCESS      Successfully get the identify controller data.
 548   @return EFI_DEVICE_ERROR Fail to get the identify controller data.
 549
 550 **/
 551 EFI_STATUS
 552 NvmeIdentifyController (
 553   IN NVME_CONTROLLER_PRIVATE_DATA       *Private,
 554   IN VOID                               *Buffer
 555   )
 556 {
 557   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
 558   NVM_EXPRESS_COMMAND                      Command;
 559   NVM_EXPRESS_RESPONSE                     Response;
 560   EFI_STATUS                               Status;
 561
 562   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 563   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
 564   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
 565
 566   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_OPC;
 567   Command.Cdw0.Cid    = Private->Cid[0]++;
 568   //
 569   // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
 570   // For the Identify command, the Namespace Identifier is only used for the Namespace data structure.
 571   //
 572   Command.Nsid        = 0;
 573
 574   CommandPacket.NvmeCmd        = &Command;
 575   CommandPacket.NvmeResponse   = &Response;
 576   CommandPacket.TransferBuffer = Buffer;
 577   CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA);
 578   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 579   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
 580   //
 581   // Set bit 0 (Cns bit) to 1 to identify a controller
 582   //
 583   Command.Cdw10                = 1;
 584   Command.Flags                = CDW10_VALID;
 585
 586   Status = Private->Passthru.PassThru (
 587                                &Private->Passthru,
 588                                NVME_CONTROLLER_ID,
 589                                0,
 590                                &CommandPacket,
 591                                NULL
 592                                );
 593
 594   return Status;
 595 }
 596
 597 /**
 598   Get specified identify namespace data.
 599
 600   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 601   @param  NamespaceId      The specified namespace identifier.
 602   @param  Buffer           The buffer used to store the identify namespace data.
 603
 604   @return EFI_SUCCESS      Successfully get the identify namespace data.
 605   @return EFI_DEVICE_ERROR Fail to get the identify namespace data.
 606
 607 **/
 608 EFI_STATUS
 609 NvmeIdentifyNamespace (
 610   IN NVME_CONTROLLER_PRIVATE_DATA      *Private,
 611   IN UINT32                            NamespaceId,
 612   IN VOID                              *Buffer
 613   )
 614 {
 615   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
 616   NVM_EXPRESS_COMMAND                      Command;
 617   NVM_EXPRESS_RESPONSE                     Response;
 618   EFI_STATUS                               Status;
 619
 620   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 621   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
 622   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
 623
 624   CommandPacket.NvmeCmd      = &Command;
 625   CommandPacket.NvmeResponse = &Response;
 626
 627   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_OPC;
 628   Command.Cdw0.Cid    = Private->Cid[0]++;
 629   Command.Nsid        = NamespaceId;
 630   CommandPacket.TransferBuffer = Buffer;
 631   CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA);
 632   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 633   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
 634   //
 635   // Set bit 0 (Cns bit) to 1 to identify a namespace
 636   //
 637   CommandPacket.NvmeCmd->Cdw10 = 0;
 638   CommandPacket.NvmeCmd->Flags = CDW10_VALID;
 639
 640   Status = Private->Passthru.PassThru (
 641                                &Private->Passthru,
 642                                NamespaceId,
 643                                0,
 644                                &CommandPacket,
 645                                NULL
 646                                );
 647
 648   return Status;
 649 }
 650
 651 /**
 652   Create io completion queue.
 653
 654   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 655
 656   @return EFI_SUCCESS      Successfully create io completion queue.
 657   @return EFI_DEVICE_ERROR Fail to create io completion queue.
 658
 659 **/
 660 EFI_STATUS
 661 NvmeCreateIoCompletionQueue (
 662   IN NVME_CONTROLLER_PRIVATE_DATA      *Private
 663   )
 664 {
 665   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
 666   NVM_EXPRESS_COMMAND                      Command;
 667   NVM_EXPRESS_RESPONSE                     Response;
 668   EFI_STATUS                               Status;
 669   NVME_ADMIN_CRIOCQ                        CrIoCq;
 670
 671   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 672   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
 673   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
 674   ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ));
 675
 676   CommandPacket.NvmeCmd      = &Command;
 677   CommandPacket.NvmeResponse = &Response;
 678
 679   Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_OPC;
 680   Command.Cdw0.Cid    = Private->Cid[0]++;
 681   CommandPacket.TransferBuffer = Private->CqBufferPciAddr[1];
 682   CommandPacket.TransferLength = EFI_PAGE_SIZE;
 683   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 684   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
 685
 686   CrIoCq.Qid   = NVME_IO_QUEUE;
 687   CrIoCq.Qsize = NVME_CCQ_SIZE;
 688   CrIoCq.Pc    = 1;
 689   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
 690   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
 691
 692   Status = Private->Passthru.PassThru (
 693                                &Private->Passthru,
 694                                0,
 695                                0,
 696                                &CommandPacket,
 697                                NULL
 698                                );
 699
 700   return Status;
 701 }
 702
 703 /**
 704   Create io submission queue.
 705
 706   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 707
 708   @return EFI_SUCCESS      Successfully create io submission queue.
 709   @return EFI_DEVICE_ERROR Fail to create io submission queue.
 710
 711 **/
 712 EFI_STATUS
 713 NvmeCreateIoSubmissionQueue (
 714   IN NVME_CONTROLLER_PRIVATE_DATA      *Private
 715   )
 716 {
 717   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
 718   NVM_EXPRESS_COMMAND                      Command;
 719   NVM_EXPRESS_RESPONSE                     Response;
 720   EFI_STATUS                               Status;
 721   NVME_ADMIN_CRIOSQ                        CrIoSq;
 722
 723   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 724   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
 725   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
 726   ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ));
 727
 728   CommandPacket.NvmeCmd      = &Command;
 729   CommandPacket.NvmeResponse = &Response;
 730
 731   Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_OPC;
 732   Command.Cdw0.Cid    = Private->Cid[0]++;
 733   CommandPacket.TransferBuffer = Private->SqBufferPciAddr[1];
 734   CommandPacket.TransferLength = EFI_PAGE_SIZE;
 735   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 736   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
 737
 738   CrIoSq.Qid   = NVME_IO_QUEUE;
 739   CrIoSq.Qsize = NVME_CSQ_SIZE;
 740   CrIoSq.Pc    = 1;
 741   CrIoSq.Cqid  = NVME_IO_QUEUE;
 742   CrIoSq.Qprio = 0;
 743   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
 744   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
 745
 746   Status = Private->Passthru.PassThru (
 747                                &Private->Passthru,
 748                                0,
 749                                0,
 750                                &CommandPacket,
 751                                NULL
 752                                );
 753
 754   return Status;
 755 }
 756
 757 /**
 758   Initialize the Nvm Express controller.
 759
 760   @param[in] Private                 The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 761
 762   @retval EFI_SUCCESS                The NVM Express Controller is initialized successfully.
 763   @retval Others                     A device error occurred while initializing the controller.
 764
 765 **/
 766 EFI_STATUS
 767 NvmeControllerInit (
 768   IN NVME_CONTROLLER_PRIVATE_DATA    *Private
 769   )
 770 {
 771   EFI_STATUS                      Status;
 772   EFI_PCI_IO_PROTOCOL             *PciIo;
 773   UINT64                          Supports;
 774   NVME_AQA                        Aqa;
 775   NVME_ASQ                        Asq;
 776   NVME_ACQ                        Acq;
 777
 778   //
 779   // Save original PCI attributes and enable this controller.
 780   //
 781   PciIo  = Private->PciIo;
 782   Status = PciIo->Attributes (
 783                     PciIo,
 784                     EfiPciIoAttributeOperationGet,
 785                     0,
 786                     &Private->PciAttributes
 787                     );
 788
 789   if (EFI_ERROR (Status)) {
 790     return Status;
 791   }
 792
 793   Status = PciIo->Attributes (
 794                     PciIo,
 795                     EfiPciIoAttributeOperationSupported,
 796                     0,
 797                     &Supports
 798                     );
 799
 800   if (!EFI_ERROR (Status)) {
 801     Supports &= EFI_PCI_DEVICE_ENABLE;
 802     Status    = PciIo->Attributes (
 803                          PciIo,
 804                          EfiPciIoAttributeOperationEnable,
 805                          Supports,
 806                          NULL
 807                          );
 808   }
 809
 810   if (EFI_ERROR (Status)) {
 811     DEBUG ((EFI_D_INFO, "NvmeControllerInit: failed to enable controller\n"));
 812     return Status;
 813   }
 814
 815   //
 816   // Read the Controller Capabilities register and verify that the NVM command set is supported
 817   //
 818   Status = ReadNvmeControllerCapabilities (Private, &Private->Cap);
 819   if (EFI_ERROR (Status)) {
 820     return Status;
 821   }
 822
 823   if (Private->Cap.Css != 0x01) {
 824     DEBUG ((EFI_D_INFO, "NvmeControllerInit: the controller doesn't support NVMe command set\n"));
 825     return EFI_UNSUPPORTED;
 826   }
 827
 828   //
 829   // Currently the driver only supports 4k page size.
 830   //
 831   ASSERT ((Private->Cap.Mpsmin + 12) <= EFI_PAGE_SHIFT);
 832
 833   Private->Cid[0] = 0;
 834   Private->Cid[1] = 0;
 835
 836   Status = NvmeDisableController (Private);
 837
 838   if (EFI_ERROR(Status)) {
 839     return Status;
 840   }
 841
 842   //
 843   // set number of entries admin submission & completion queues.
 844   //
 845   Aqa.Asqs = NVME_ASQ_SIZE;
 846   Aqa.Acqs = NVME_ACQ_SIZE;
 847
 848   //
 849   // Address of admin submission queue.
 850   //
 851   Asq.Rsvd1 = 0;
 852   Asq.Asqb  = (UINT64)(UINTN)(Private->BufferPciAddr) >> 12;
 853
 854   //
 855   // Address of admin completion queue.
 856   //
 857   Acq.Rsvd1 = 0;
 858   Acq.Acqb  = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) >> 12;
 859
 860   //
 861   // Address of I/O submission & completion queue.
 862   //
 863   Private->SqBuffer[0]        = (NVME_SQ *)(UINTN)(Private->Buffer);
 864   Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr);
 865   Private->CqBuffer[0]        = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE);
 866   Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 1 * EFI_PAGE_SIZE);
 867   Private->SqBuffer[1]        = (NVME_SQ *)(UINTN)(Private->Buffer + 2 * EFI_PAGE_SIZE);
 868   Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE);
 869   Private->CqBuffer[1]        = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE);
 870   Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE);
 871
 872   DEBUG ((EFI_D_INFO, "Private->Buffer = [%016X]\n", (UINT64)(UINTN)Private->Buffer));
 873   DEBUG ((EFI_D_INFO, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));
 874   DEBUG ((EFI_D_INFO, "Admin Completion Queue size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));
 875   DEBUG ((EFI_D_INFO, "Admin Submission Queue (SqBuffer[0]) = [%016X]\n", Private->SqBuffer[0]));
 876   DEBUG ((EFI_D_INFO, "Admin Completion Queue (CqBuffer[0]) = [%016X]\n", Private->CqBuffer[0]));
 877   DEBUG ((EFI_D_INFO, "I/O   Submission Queue (SqBuffer[1]) = [%016X]\n", Private->SqBuffer[1]));
 878   DEBUG ((EFI_D_INFO, "I/O   Completion Queue (CqBuffer[1]) = [%016X]\n", Private->CqBuffer[1]));
 879
 880   //
 881   // Program admin queue attributes.
 882   //
 883   Status = WriteNvmeAdminQueueAttributes (Private, &Aqa);
 884
 885   if (EFI_ERROR(Status)) {
 886     return Status;
 887   }
 888
 889   //
 890   // Program admin submission queue address.
 891   //
 892   Status = WriteNvmeAdminSubmissionQueueBaseAddress (Private, &Asq);
 893
 894   if (EFI_ERROR(Status)) {
 895     return Status;
 896   }
 897
 898   //
 899   // Program admin completion queue address.
 900   //
 901   Status = WriteNvmeAdminCompletionQueueBaseAddress (Private, &Acq);
 902
 903   if (EFI_ERROR(Status)) {
 904     return Status;
 905   }
 906
 907   Status = NvmeEnableController (Private);
 908   if (EFI_ERROR(Status)) {
 909     return Status;
 910   }
 911
 912   //
 913   // Create one I/O completion queue.
 914   //
 915   Status = NvmeCreateIoCompletionQueue (Private);
 916   if (EFI_ERROR(Status)) {
 917    return Status;
 918   }
 919
 920   //
 921   // Create one I/O Submission queue.
 922   //
 923   Status = NvmeCreateIoSubmissionQueue (Private);
 924   if (EFI_ERROR(Status)) {
 925    return Status;
 926   }
 927
 928   //
 929   // Allocate buffer for Identify Controller data
 930   //
 931   Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof(NVME_ADMIN_CONTROLLER_DATA));
 932
 933   if (Private->ControllerData == NULL) {
 934     return EFI_OUT_OF_RESOURCES;
 935   }
 936
 937   //
 938   // Get current Identify Controller Data
 939   //
 940   Status = NvmeIdentifyController (Private, Private->ControllerData);
 941
 942   if (EFI_ERROR(Status)) {
 943     FreePool(Private->ControllerData);
 944     Private->ControllerData = NULL;
 945     return EFI_NOT_FOUND;
 946   }
 947
 948   //
 949   // Dump NvmExpress Identify Controller Data
 950   //
 951   Private->ControllerData->Sn[19] = 0;
 952   Private->ControllerData->Mn[39] = 0;
 953   DEBUG ((EFI_D_INFO, " == NVME IDENTIFY CONTROLLER DATA ==\n"));
 954   DEBUG ((EFI_D_INFO, "    PCI VID   : 0x%x\n", Private->ControllerData->Vid));
 955   DEBUG ((EFI_D_INFO, "    PCI SSVID : 0x%x\n", Private->ControllerData->Ssvid));
 956   DEBUG ((EFI_D_INFO, "    SN        : %a\n",   (CHAR8 *)(Private->ControllerData->Sn)));
 957   DEBUG ((EFI_D_INFO, "    MN        : %a\n",   (CHAR8 *)(Private->ControllerData->Mn)));
 958   DEBUG ((EFI_D_INFO, "    FR        : 0x%x\n", *((UINT64*)Private->ControllerData->Fr)));
 959   DEBUG ((EFI_D_INFO, "    RAB       : 0x%x\n", Private->ControllerData->Rab));
 960   DEBUG ((EFI_D_INFO, "    IEEE      : 0x%x\n", *(UINT32*)Private->ControllerData->Ieee_oiu));
 961   DEBUG ((EFI_D_INFO, "    AERL      : 0x%x\n", Private->ControllerData->Aerl));
 962   DEBUG ((EFI_D_INFO, "    SQES      : 0x%x\n", Private->ControllerData->Sqes));
 963   DEBUG ((EFI_D_INFO, "    CQES      : 0x%x\n", Private->ControllerData->Cqes));
 964   DEBUG ((EFI_D_INFO, "    NN        : 0x%x\n", Private->ControllerData->Nn));
 965
 966   return Status;
 967 }
 968