MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c

   1 /** @file
   2   NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
   3   NVM Express specification.
   4
   5   Copyright (c) 2013 - 2014, Intel Corporation. All rights reserved.<BR>
   6   This program and the accompanying materials
   7   are licensed and made available under the terms and conditions of the BSD License
   8   which accompanies this distribution.  The full text of the license may be found at
   9   http://opensource.org/licenses/bsd-license.php.
  10
  11   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13
  14 **/
  15
  16 #include "NvmExpress.h"
  17
  18 /**
  19   Read Nvm Express controller capability register.
  20
  21   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
  22   @param  Cap              The buffer used to store capability register content.
  23
  24   @return EFI_SUCCESS      Successfully read the controller capability register content.
  25   @return EFI_DEVICE_ERROR Fail to read the controller capability register.
  26
  27 **/
  28 EFI_STATUS
  29 ReadNvmeControllerCapabilities (
  30   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
  31   IN NVME_CAP                         *Cap
  32   )
  33 {
  34   EFI_PCI_IO_PROTOCOL   *PciIo;
  35   EFI_STATUS            Status;
  36   UINT64                Data;
  37
  38   PciIo  = Private->PciIo;
  39   Status = PciIo->Mem.Read (
  40                         PciIo,
  41                         EfiPciIoWidthUint32,
  42                         NVME_BAR,
  43                         NVME_CAP_OFFSET,
  44                         2,
  45                         &Data
  46                         );
  47
  48   if (EFI_ERROR(Status)) {
  49     return Status;
  50   }
  51
  52   WriteUnaligned64 ((UINT64*)Cap, Data);
  53   return EFI_SUCCESS;
  54 }
  55
  56 /**
  57   Read Nvm Express controller configuration register.
  58
  59   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
  60   @param  Cc               The buffer used to store configuration register content.
  61
  62   @return EFI_SUCCESS      Successfully read the controller configuration register content.
  63   @return EFI_DEVICE_ERROR Fail to read the controller configuration register.
  64
  65 **/
  66 EFI_STATUS
  67 ReadNvmeControllerConfiguration (
  68   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
  69   IN NVME_CC                          *Cc
  70   )
  71 {
  72   EFI_PCI_IO_PROTOCOL   *PciIo;
  73   EFI_STATUS            Status;
  74   UINT32                Data;
  75
  76   PciIo  = Private->PciIo;
  77   Status = PciIo->Mem.Read (
  78                         PciIo,
  79                         EfiPciIoWidthUint32,
  80                         NVME_BAR,
  81                         NVME_CC_OFFSET,
  82                         1,
  83                         &Data
  84                         );
  85
  86   if (EFI_ERROR(Status)) {
  87     return Status;
  88   }
  89
  90   WriteUnaligned32 ((UINT32*)Cc, Data);
  91   return EFI_SUCCESS;
  92 }
  93
  94 /**
  95   Write Nvm Express controller configuration register.
  96
  97   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
  98   @param  Cc               The buffer used to store the content to be written into configuration register.
  99
 100   @return EFI_SUCCESS      Successfully write data into the controller configuration register.
 101   @return EFI_DEVICE_ERROR Fail to write data into the controller configuration register.
 102
 103 **/
 104 EFI_STATUS
 105 WriteNvmeControllerConfiguration (
 106   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
 107   IN NVME_CC                          *Cc
 108   )
 109 {
 110   EFI_PCI_IO_PROTOCOL   *PciIo;
 111   EFI_STATUS            Status;
 112   UINT32                Data;
 113
 114   PciIo  = Private->PciIo;
 115   Data   = ReadUnaligned32 ((UINT32*)Cc);
 116   Status = PciIo->Mem.Write (
 117                         PciIo,
 118                         EfiPciIoWidthUint32,
 119                         NVME_BAR,
 120                         NVME_CC_OFFSET,
 121                         1,
 122                         &Data
 123                         );
 124
 125   if (EFI_ERROR(Status)) {
 126     return Status;
 127   }
 128
 129   DEBUG ((EFI_D_INFO, "Cc.En: %d\n", Cc->En));
 130   DEBUG ((EFI_D_INFO, "Cc.Css: %d\n", Cc->Css));
 131   DEBUG ((EFI_D_INFO, "Cc.Mps: %d\n", Cc->Mps));
 132   DEBUG ((EFI_D_INFO, "Cc.Ams: %d\n", Cc->Ams));
 133   DEBUG ((EFI_D_INFO, "Cc.Shn: %d\n", Cc->Shn));
 134   DEBUG ((EFI_D_INFO, "Cc.Iosqes: %d\n", Cc->Iosqes));
 135   DEBUG ((EFI_D_INFO, "Cc.Iocqes: %d\n", Cc->Iocqes));
 136
 137   return EFI_SUCCESS;
 138 }
 139
 140 /**
 141   Read Nvm Express controller status register.
 142
 143   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 144   @param  Csts             The buffer used to store status register content.
 145
 146   @return EFI_SUCCESS      Successfully read the controller status register content.
 147   @return EFI_DEVICE_ERROR Fail to read the controller status register.
 148
 149 **/
 150 EFI_STATUS
 151 ReadNvmeControllerStatus (
 152   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
 153   IN NVME_CSTS                        *Csts
 154   )
 155 {
 156   EFI_PCI_IO_PROTOCOL   *PciIo;
 157   EFI_STATUS            Status;
 158   UINT32                Data;
 159
 160   PciIo  = Private->PciIo;
 161   Status = PciIo->Mem.Read (
 162                         PciIo,
 163                         EfiPciIoWidthUint32,
 164                         NVME_BAR,
 165                         NVME_CSTS_OFFSET,
 166                         1,
 167                         &Data
 168                         );
 169
 170   if (EFI_ERROR(Status)) {
 171     return Status;
 172   }
 173
 174   WriteUnaligned32 ((UINT32*)Csts, Data);
 175   return EFI_SUCCESS;
 176 }
 177
 178 /**
 179   Read Nvm Express admin queue attributes register.
 180
 181   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 182   @param  Aqa              The buffer used to store admin queue attributes register content.
 183
 184   @return EFI_SUCCESS      Successfully read the admin queue attributes register content.
 185   @return EFI_DEVICE_ERROR Fail to read the admin queue attributes register.
 186
 187 **/
 188 EFI_STATUS
 189 ReadNvmeAdminQueueAttributes (
 190   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
 191   IN NVME_AQA                         *Aqa
 192   )
 193 {
 194   EFI_PCI_IO_PROTOCOL   *PciIo;
 195   EFI_STATUS            Status;
 196   UINT32                Data;
 197
 198   PciIo  = Private->PciIo;
 199   Status = PciIo->Mem.Read (
 200                         PciIo,
 201                         EfiPciIoWidthUint32,
 202                         NVME_BAR,
 203                         NVME_AQA_OFFSET,
 204                         1,
 205                         &Data
 206                         );
 207
 208   if (EFI_ERROR(Status)) {
 209     return Status;
 210   }
 211
 212   WriteUnaligned32 ((UINT32*)Aqa, Data);
 213   return EFI_SUCCESS;
 214 }
 215
 216 /**
 217   Write Nvm Express admin queue attributes register.
 218
 219   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 220   @param  Aqa              The buffer used to store the content to be written into admin queue attributes register.
 221
 222   @return EFI_SUCCESS      Successfully write data into the admin queue attributes register.
 223   @return EFI_DEVICE_ERROR Fail to write data into the admin queue attributes register.
 224
 225 **/
 226 EFI_STATUS
 227 WriteNvmeAdminQueueAttributes (
 228   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
 229   IN NVME_AQA                         *Aqa
 230   )
 231 {
 232   EFI_PCI_IO_PROTOCOL   *PciIo;
 233   EFI_STATUS            Status;
 234   UINT32                Data;
 235
 236   PciIo  = Private->PciIo;
 237   Data   = ReadUnaligned32 ((UINT32*)Aqa);
 238   Status = PciIo->Mem.Write (
 239                         PciIo,
 240                         EfiPciIoWidthUint32,
 241                         NVME_BAR,
 242                         NVME_AQA_OFFSET,
 243                         1,
 244                         &Data
 245                         );
 246
 247   if (EFI_ERROR(Status)) {
 248     return Status;
 249   }
 250
 251   DEBUG ((EFI_D_INFO, "Aqa.Asqs: %d\n", Aqa->Asqs));
 252   DEBUG ((EFI_D_INFO, "Aqa.Acqs: %d\n", Aqa->Acqs));
 253
 254   return EFI_SUCCESS;
 255 }
 256
 257 /**
 258   Read Nvm Express admin submission queue base address register.
 259
 260   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 261   @param  Asq              The buffer used to store admin submission queue base address register content.
 262
 263   @return EFI_SUCCESS      Successfully read the admin submission queue base address register content.
 264   @return EFI_DEVICE_ERROR Fail to read the admin submission queue base address register.
 265
 266 **/
 267 EFI_STATUS
 268 ReadNvmeAdminSubmissionQueueBaseAddress (
 269   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
 270   IN NVME_ASQ                         *Asq
 271   )
 272 {
 273   EFI_PCI_IO_PROTOCOL   *PciIo;
 274   EFI_STATUS            Status;
 275   UINT64                Data;
 276
 277   PciIo  = Private->PciIo;
 278   Status = PciIo->Mem.Read (
 279                         PciIo,
 280                         EfiPciIoWidthUint32,
 281                         NVME_BAR,
 282                         NVME_ASQ_OFFSET,
 283                         2,
 284                         &Data
 285                         );
 286
 287   if (EFI_ERROR(Status)) {
 288     return Status;
 289   }
 290
 291   WriteUnaligned64 ((UINT64*)Asq, Data);
 292   return EFI_SUCCESS;
 293 }
 294
 295 /**
 296   Write Nvm Express admin submission queue base address register.
 297
 298   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 299   @param  Asq              The buffer used to store the content to be written into admin submission queue base address register.
 300
 301   @return EFI_SUCCESS      Successfully write data into the admin submission queue base address register.
 302   @return EFI_DEVICE_ERROR Fail to write data into the admin submission queue base address register.
 303
 304 **/
 305 EFI_STATUS
 306 WriteNvmeAdminSubmissionQueueBaseAddress (
 307   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
 308   IN NVME_ASQ                         *Asq
 309   )
 310 {
 311   EFI_PCI_IO_PROTOCOL   *PciIo;
 312   EFI_STATUS            Status;
 313   UINT64                Data;
 314
 315   PciIo  = Private->PciIo;
 316   Data   = ReadUnaligned64 ((UINT64*)Asq);
 317
 318   Status = PciIo->Mem.Write (
 319                         PciIo,
 320                         EfiPciIoWidthUint32,
 321                         NVME_BAR,
 322                         NVME_ASQ_OFFSET,
 323                         2,
 324                         &Data
 325                         );
 326
 327   if (EFI_ERROR(Status)) {
 328     return Status;
 329   }
 330
 331   DEBUG ((EFI_D_INFO, "Asq.Asqb: %lx\n", Asq->Asqb));
 332
 333   return EFI_SUCCESS;
 334 }
 335
 336 /**
 337   Read Nvm Express admin completion queue base address register.
 338
 339   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 340   @param  Acq              The buffer used to store admin completion queue base address register content.
 341
 342   @return EFI_SUCCESS      Successfully read the admin completion queue base address register content.
 343   @return EFI_DEVICE_ERROR Fail to read the admin completion queue base address register.
 344
 345 **/
 346 EFI_STATUS
 347 ReadNvmeAdminCompletionQueueBaseAddress (
 348   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
 349   IN NVME_ACQ                         *Acq
 350   )
 351 {
 352   EFI_PCI_IO_PROTOCOL   *PciIo;
 353   EFI_STATUS            Status;
 354   UINT64                Data;
 355
 356   PciIo  = Private->PciIo;
 357
 358   Status = PciIo->Mem.Read (
 359                         PciIo,
 360                         EfiPciIoWidthUint32,
 361                         NVME_BAR,
 362                         NVME_ACQ_OFFSET,
 363                         2,
 364                         &Data
 365                         );
 366
 367   if (EFI_ERROR(Status)) {
 368     return Status;
 369   }
 370
 371   WriteUnaligned64 ((UINT64*)Acq, Data);
 372   return EFI_SUCCESS;
 373 }
 374
 375 /**
 376   Write Nvm Express admin completion queue base address register.
 377
 378   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 379   @param  Acq              The buffer used to store the content to be written into admin completion queue base address register.
 380
 381   @return EFI_SUCCESS      Successfully write data into the admin completion queue base address register.
 382   @return EFI_DEVICE_ERROR Fail to write data into the admin completion queue base address register.
 383
 384 **/
 385 EFI_STATUS
 386 WriteNvmeAdminCompletionQueueBaseAddress (
 387   IN NVME_CONTROLLER_PRIVATE_DATA     *Private,
 388   IN NVME_ACQ                         *Acq
 389   )
 390 {
 391   EFI_PCI_IO_PROTOCOL   *PciIo;
 392   EFI_STATUS            Status;
 393   UINT64                Data;
 394
 395   PciIo  = Private->PciIo;
 396   Data   = ReadUnaligned64 ((UINT64*)Acq);
 397
 398   Status = PciIo->Mem.Write (
 399                         PciIo,
 400                         EfiPciIoWidthUint32,
 401                         NVME_BAR,
 402                         NVME_ACQ_OFFSET,
 403                         2,
 404                         &Data
 405                         );
 406
 407   if (EFI_ERROR(Status)) {
 408     return Status;
 409   }
 410
 411   DEBUG ((EFI_D_INFO, "Acq.Acqb: %lxh\n", Acq->Acqb));
 412
 413   return EFI_SUCCESS;
 414 }
 415
 416 /**
 417   Disable the Nvm Express controller.
 418
 419   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 420
 421   @return EFI_SUCCESS      Successfully disable the controller.
 422   @return EFI_DEVICE_ERROR Fail to disable the controller.
 423
 424 **/
 425 EFI_STATUS
 426 NvmeDisableController (
 427   IN NVME_CONTROLLER_PRIVATE_DATA     *Private
 428   )
 429 {
 430   NVME_CC                Cc;
 431   NVME_CSTS              Csts;
 432   EFI_STATUS             Status;
 433   UINT32                 Index;
 434   UINT8                  Timeout;
 435
 436   //
 437   // Read Controller Configuration Register.
 438   //
 439   Status = ReadNvmeControllerConfiguration (Private, &Cc);
 440   if (EFI_ERROR(Status)) {
 441     return Status;
 442   }
 443
 444   Cc.En = 0;
 445
 446   //
 447   // Disable the controller.
 448   //
 449   Status = WriteNvmeControllerConfiguration (Private, &Cc);
 450
 451   if (EFI_ERROR(Status)) {
 452     return Status;
 453   }
 454
 455   //
 456   // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to transition from 1 to 0 after
 457   // Cc.Enable transition from 1 to 0. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
 458   //
 459   if (Private->Cap.To == 0) {
 460     Timeout = 1;
 461   } else {
 462     Timeout = Private->Cap.To;
 463   }
 464
 465   for(Index = (Timeout * 500); Index != 0; --Index) {
 466     gBS->Stall(1000);
 467
 468     //
 469     // Check if the controller is initialized
 470     //
 471     Status = ReadNvmeControllerStatus (Private, &Csts);
 472
 473     if (EFI_ERROR(Status)) {
 474       return Status;
 475     }
 476
 477     if (Csts.Rdy == 0) {
 478       break;
 479     }
 480   }
 481
 482   if (Index == 0) {
 483     Status = EFI_DEVICE_ERROR;
 484   }
 485
 486   DEBUG ((EFI_D_INFO, "NVMe controller is disabled with status [%r].\n", Status));
 487   return Status;
 488 }
 489
 490 /**
 491   Enable the Nvm Express controller.
 492
 493   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 494
 495   @return EFI_SUCCESS      Successfully enable the controller.
 496   @return EFI_DEVICE_ERROR Fail to enable the controller.
 497   @return EFI_TIMEOUT      Fail to enable the controller in given time slot.
 498
 499 **/
 500 EFI_STATUS
 501 NvmeEnableController (
 502   IN NVME_CONTROLLER_PRIVATE_DATA     *Private
 503   )
 504 {
 505   NVME_CC                Cc;
 506   NVME_CSTS              Csts;
 507   EFI_STATUS             Status;
 508   UINT32                 Index;
 509   UINT8                  Timeout;
 510
 511   //
 512   // Enable the controller
 513   //
 514   ZeroMem (&Cc, sizeof (NVME_CC));
 515   Cc.En     = 1;
 516   Cc.Iosqes = 6;
 517   Cc.Iocqes = 4;
 518   Status    = WriteNvmeControllerConfiguration (Private, &Cc);
 519
 520   if (EFI_ERROR(Status)) {
 521     return Status;
 522   }
 523
 524   //
 525   // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
 526   // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
 527   //
 528   if (Private->Cap.To == 0) {
 529     Timeout = 1;
 530   } else {
 531     Timeout = Private->Cap.To;
 532   }
 533
 534   for(Index = (Timeout * 500); Index != 0; --Index) {
 535     gBS->Stall(1000);
 536
 537     //
 538     // Check if the controller is initialized
 539     //
 540     Status = ReadNvmeControllerStatus (Private, &Csts);
 541
 542     if (EFI_ERROR(Status)) {
 543       return Status;
 544     }
 545
 546     if (Csts.Rdy) {
 547       break;
 548     }
 549   }
 550
 551   if (Index == 0) {
 552     Status = EFI_TIMEOUT;
 553   }
 554
 555   DEBUG ((EFI_D_INFO, "NVMe controller is enabled with status [%r].\n", Status));
 556   return Status;
 557 }
 558
 559 /**
 560   Get identify controller data.
 561
 562   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 563   @param  Buffer           The buffer used to store the identify controller data.
 564
 565   @return EFI_SUCCESS      Successfully get the identify controller data.
 566   @return EFI_DEVICE_ERROR Fail to get the identify controller data.
 567
 568 **/
 569 EFI_STATUS
 570 NvmeIdentifyController (
 571   IN NVME_CONTROLLER_PRIVATE_DATA       *Private,
 572   IN VOID                               *Buffer
 573   )
 574 {
 575   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
 576   NVM_EXPRESS_COMMAND                      Command;
 577   NVM_EXPRESS_RESPONSE                     Response;
 578   EFI_STATUS                               Status;
 579
 580   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 581   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
 582   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
 583
 584   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_OPC;
 585   Command.Cdw0.Cid    = Private->Cid[0]++;
 586   //
 587   // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
 588   // For the Identify command, the Namespace Identifier is only used for the Namespace data structure.
 589   //
 590   Command.Nsid        = 0;
 591
 592   CommandPacket.NvmeCmd        = &Command;
 593   CommandPacket.NvmeResponse   = &Response;
 594   CommandPacket.TransferBuffer = Buffer;
 595   CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA);
 596   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 597   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
 598   //
 599   // Set bit 0 (Cns bit) to 1 to identify a controller
 600   //
 601   Command.Cdw10                = 1;
 602   Command.Flags                = CDW10_VALID;
 603
 604   Status = Private->Passthru.PassThru (
 605                                &Private->Passthru,
 606                                NVME_CONTROLLER_ID,
 607                                0,
 608                                &CommandPacket,
 609                                NULL
 610                                );
 611
 612   return Status;
 613 }
 614
 615 /**
 616   Get specified identify namespace data.
 617
 618   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 619   @param  NamespaceId      The specified namespace identifier.
 620   @param  Buffer           The buffer used to store the identify namespace data.
 621
 622   @return EFI_SUCCESS      Successfully get the identify namespace data.
 623   @return EFI_DEVICE_ERROR Fail to get the identify namespace data.
 624
 625 **/
 626 EFI_STATUS
 627 NvmeIdentifyNamespace (
 628   IN NVME_CONTROLLER_PRIVATE_DATA      *Private,
 629   IN UINT32                            NamespaceId,
 630   IN VOID                              *Buffer
 631   )
 632 {
 633   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
 634   NVM_EXPRESS_COMMAND                      Command;
 635   NVM_EXPRESS_RESPONSE                     Response;
 636   EFI_STATUS                               Status;
 637
 638   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 639   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
 640   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
 641
 642   CommandPacket.NvmeCmd      = &Command;
 643   CommandPacket.NvmeResponse = &Response;
 644
 645   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_OPC;
 646   Command.Cdw0.Cid    = Private->Cid[0]++;
 647   Command.Nsid        = NamespaceId;
 648   CommandPacket.TransferBuffer = Buffer;
 649   CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA);
 650   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 651   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
 652   //
 653   // Set bit 0 (Cns bit) to 1 to identify a namespace
 654   //
 655   CommandPacket.NvmeCmd->Cdw10 = 0;
 656   CommandPacket.NvmeCmd->Flags = CDW10_VALID;
 657
 658   Status = Private->Passthru.PassThru (
 659                                &Private->Passthru,
 660                                NamespaceId,
 661                                0,
 662                                &CommandPacket,
 663                                NULL
 664                                );
 665
 666   return Status;
 667 }
 668
 669 /**
 670   Create io completion queue.
 671
 672   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 673
 674   @return EFI_SUCCESS      Successfully create io completion queue.
 675   @return EFI_DEVICE_ERROR Fail to create io completion queue.
 676
 677 **/
 678 EFI_STATUS
 679 NvmeCreateIoCompletionQueue (
 680   IN NVME_CONTROLLER_PRIVATE_DATA      *Private
 681   )
 682 {
 683   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
 684   NVM_EXPRESS_COMMAND                      Command;
 685   NVM_EXPRESS_RESPONSE                     Response;
 686   EFI_STATUS                               Status;
 687   NVME_ADMIN_CRIOCQ                        CrIoCq;
 688
 689   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 690   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
 691   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
 692   ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ));
 693
 694   CommandPacket.NvmeCmd      = &Command;
 695   CommandPacket.NvmeResponse = &Response;
 696
 697   Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_OPC;
 698   Command.Cdw0.Cid    = Private->Cid[0]++;
 699   CommandPacket.TransferBuffer = Private->CqBufferPciAddr[1];
 700   CommandPacket.TransferLength = EFI_PAGE_SIZE;
 701   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 702   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
 703
 704   CrIoCq.Qid   = NVME_IO_QUEUE;
 705   CrIoCq.Qsize = NVME_CCQ_SIZE;
 706   CrIoCq.Pc    = 1;
 707   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
 708   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
 709
 710   Status = Private->Passthru.PassThru (
 711                                &Private->Passthru,
 712                                0,
 713                                0,
 714                                &CommandPacket,
 715                                NULL
 716                                );
 717
 718   return Status;
 719 }
 720
 721 /**
 722   Create io submission queue.
 723
 724   @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 725
 726   @return EFI_SUCCESS      Successfully create io submission queue.
 727   @return EFI_DEVICE_ERROR Fail to create io submission queue.
 728
 729 **/
 730 EFI_STATUS
 731 NvmeCreateIoSubmissionQueue (
 732   IN NVME_CONTROLLER_PRIVATE_DATA      *Private
 733   )
 734 {
 735   NVM_EXPRESS_PASS_THRU_COMMAND_PACKET     CommandPacket;
 736   NVM_EXPRESS_COMMAND                      Command;
 737   NVM_EXPRESS_RESPONSE                     Response;
 738   EFI_STATUS                               Status;
 739   NVME_ADMIN_CRIOSQ                        CrIoSq;
 740
 741   ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 742   ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
 743   ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
 744   ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ));
 745
 746   CommandPacket.NvmeCmd      = &Command;
 747   CommandPacket.NvmeResponse = &Response;
 748
 749   Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_OPC;
 750   Command.Cdw0.Cid    = Private->Cid[0]++;
 751   CommandPacket.TransferBuffer = Private->SqBufferPciAddr[1];
 752   CommandPacket.TransferLength = EFI_PAGE_SIZE;
 753   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 754   CommandPacket.QueueId        = NVME_ADMIN_QUEUE;
 755
 756   CrIoSq.Qid   = NVME_IO_QUEUE;
 757   CrIoSq.Qsize = NVME_CSQ_SIZE;
 758   CrIoSq.Pc    = 1;
 759   CrIoSq.Cqid  = NVME_IO_QUEUE;
 760   CrIoSq.Qprio = 0;
 761   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
 762   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
 763
 764   Status = Private->Passthru.PassThru (
 765                                &Private->Passthru,
 766                                0,
 767                                0,
 768                                &CommandPacket,
 769                                NULL
 770                                );
 771
 772   return Status;
 773 }
 774
 775 /**
 776   Initialize the Nvm Express controller.
 777
 778   @param[in] Private                 The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 779
 780   @retval EFI_SUCCESS                The NVM Express Controller is initialized successfully.
 781   @retval Others                     A device error occurred while initializing the controller.
 782
 783 **/
 784 EFI_STATUS
 785 NvmeControllerInit (
 786   IN NVME_CONTROLLER_PRIVATE_DATA    *Private
 787   )
 788 {
 789   EFI_STATUS                      Status;
 790   EFI_PCI_IO_PROTOCOL             *PciIo;
 791   UINT64                          Supports;
 792   NVME_AQA                        Aqa;
 793   NVME_ASQ                        Asq;
 794   NVME_ACQ                        Acq;
 795
 796   //
 797   // Save original PCI attributes and enable this controller.
 798   //
 799   PciIo  = Private->PciIo;
 800   Status = PciIo->Attributes (
 801                     PciIo,
 802                     EfiPciIoAttributeOperationGet,
 803                     0,
 804                     &Private->PciAttributes
 805                     );
 806
 807   if (EFI_ERROR (Status)) {
 808     return Status;
 809   }
 810
 811   Status = PciIo->Attributes (
 812                     PciIo,
 813                     EfiPciIoAttributeOperationSupported,
 814                     0,
 815                     &Supports
 816                     );
 817
 818   if (!EFI_ERROR (Status)) {
 819     Supports &= EFI_PCI_DEVICE_ENABLE;
 820     Status    = PciIo->Attributes (
 821                          PciIo,
 822                          EfiPciIoAttributeOperationEnable,
 823                          Supports,
 824                          NULL
 825                          );
 826   }
 827
 828   if (EFI_ERROR (Status)) {
 829     DEBUG ((EFI_D_INFO, "NvmeControllerInit: failed to enable controller\n"));
 830     return Status;
 831   }
 832
 833   //
 834   // Read the Controller Capabilities register and verify that the NVM command set is supported
 835   //
 836   Status = ReadNvmeControllerCapabilities (Private, &Private->Cap);
 837   if (EFI_ERROR (Status)) {
 838     return Status;
 839   }
 840
 841   if (Private->Cap.Css != 0x01) {
 842     DEBUG ((EFI_D_INFO, "NvmeControllerInit: the controller doesn't support NVMe command set\n"));
 843     return EFI_UNSUPPORTED;
 844   }
 845
 846   //
 847   // Currently the driver only supports 4k page size.
 848   //
 849   ASSERT ((Private->Cap.Mpsmin + 12) <= EFI_PAGE_SHIFT);
 850
 851   Private->Cid[0] = 0;
 852   Private->Cid[1] = 0;
 853
 854   Status = NvmeDisableController (Private);
 855
 856   if (EFI_ERROR(Status)) {
 857     return Status;
 858   }
 859
 860   //
 861   // set number of entries admin submission & completion queues.
 862   //
 863   Aqa.Asqs  = NVME_ASQ_SIZE;
 864   Aqa.Rsvd1 = 0;
 865   Aqa.Acqs  = NVME_ACQ_SIZE;
 866   Aqa.Rsvd2 = 0;
 867
 868   //
 869   // Address of admin submission queue.
 870   //
 871   Asq.Rsvd1 = 0;
 872   Asq.Asqb  = (UINT64)(UINTN)(Private->BufferPciAddr) >> 12;
 873
 874   //
 875   // Address of admin completion queue.
 876   //
 877   Acq.Rsvd1 = 0;
 878   Acq.Acqb  = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) >> 12;
 879
 880   //
 881   // Address of I/O submission & completion queue.
 882   //
 883   Private->SqBuffer[0]        = (NVME_SQ *)(UINTN)(Private->Buffer);
 884   Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr);
 885   Private->CqBuffer[0]        = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE);
 886   Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 1 * EFI_PAGE_SIZE);
 887   Private->SqBuffer[1]        = (NVME_SQ *)(UINTN)(Private->Buffer + 2 * EFI_PAGE_SIZE);
 888   Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE);
 889   Private->CqBuffer[1]        = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE);
 890   Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE);
 891
 892   DEBUG ((EFI_D_INFO, "Private->Buffer = [%016X]\n", (UINT64)(UINTN)Private->Buffer));
 893   DEBUG ((EFI_D_INFO, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));
 894   DEBUG ((EFI_D_INFO, "Admin Completion Queue size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));
 895   DEBUG ((EFI_D_INFO, "Admin Submission Queue (SqBuffer[0]) = [%016X]\n", Private->SqBuffer[0]));
 896   DEBUG ((EFI_D_INFO, "Admin Completion Queue (CqBuffer[0]) = [%016X]\n", Private->CqBuffer[0]));
 897   DEBUG ((EFI_D_INFO, "I/O   Submission Queue (SqBuffer[1]) = [%016X]\n", Private->SqBuffer[1]));
 898   DEBUG ((EFI_D_INFO, "I/O   Completion Queue (CqBuffer[1]) = [%016X]\n", Private->CqBuffer[1]));
 899
 900   //
 901   // Program admin queue attributes.
 902   //
 903   Status = WriteNvmeAdminQueueAttributes (Private, &Aqa);
 904
 905   if (EFI_ERROR(Status)) {
 906     return Status;
 907   }
 908
 909   //
 910   // Program admin submission queue address.
 911   //
 912   Status = WriteNvmeAdminSubmissionQueueBaseAddress (Private, &Asq);
 913
 914   if (EFI_ERROR(Status)) {
 915     return Status;
 916   }
 917
 918   //
 919   // Program admin completion queue address.
 920   //
 921   Status = WriteNvmeAdminCompletionQueueBaseAddress (Private, &Acq);
 922
 923   if (EFI_ERROR(Status)) {
 924     return Status;
 925   }
 926
 927   Status = NvmeEnableController (Private);
 928   if (EFI_ERROR(Status)) {
 929     return Status;
 930   }
 931
 932   //
 933   // Create one I/O completion queue.
 934   //
 935   Status = NvmeCreateIoCompletionQueue (Private);
 936   if (EFI_ERROR(Status)) {
 937    return Status;
 938   }
 939
 940   //
 941   // Create one I/O Submission queue.
 942   //
 943   Status = NvmeCreateIoSubmissionQueue (Private);
 944   if (EFI_ERROR(Status)) {
 945    return Status;
 946   }
 947
 948   //
 949   // Allocate buffer for Identify Controller data
 950   //
 951   Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof(NVME_ADMIN_CONTROLLER_DATA));
 952
 953   if (Private->ControllerData == NULL) {
 954     return EFI_OUT_OF_RESOURCES;
 955   }
 956
 957   //
 958   // Get current Identify Controller Data
 959   //
 960   Status = NvmeIdentifyController (Private, Private->ControllerData);
 961
 962   if (EFI_ERROR(Status)) {
 963     FreePool(Private->ControllerData);
 964     Private->ControllerData = NULL;
 965     return EFI_NOT_FOUND;
 966   }
 967
 968   //
 969   // Dump NvmExpress Identify Controller Data
 970   //
 971   Private->ControllerData->Sn[19] = 0;
 972   Private->ControllerData->Mn[39] = 0;
 973   DEBUG ((EFI_D_INFO, " == NVME IDENTIFY CONTROLLER DATA ==\n"));
 974   DEBUG ((EFI_D_INFO, "    PCI VID   : 0x%x\n", Private->ControllerData->Vid));
 975   DEBUG ((EFI_D_INFO, "    PCI SSVID : 0x%x\n", Private->ControllerData->Ssvid));
 976   DEBUG ((EFI_D_INFO, "    SN        : %a\n",   (CHAR8 *)(Private->ControllerData->Sn)));
 977   DEBUG ((EFI_D_INFO, "    MN        : %a\n",   (CHAR8 *)(Private->ControllerData->Mn)));
 978   DEBUG ((EFI_D_INFO, "    FR        : 0x%x\n", *((UINT64*)Private->ControllerData->Fr)));
 979   DEBUG ((EFI_D_INFO, "    RAB       : 0x%x\n", Private->ControllerData->Rab));
 980   DEBUG ((EFI_D_INFO, "    IEEE      : 0x%x\n", *(UINT32*)Private->ControllerData->Ieee_oui));
 981   DEBUG ((EFI_D_INFO, "    AERL      : 0x%x\n", Private->ControllerData->Aerl));
 982   DEBUG ((EFI_D_INFO, "    SQES      : 0x%x\n", Private->ControllerData->Sqes));
 983   DEBUG ((EFI_D_INFO, "    CQES      : 0x%x\n", Private->ControllerData->Cqes));
 984   DEBUG ((EFI_D_INFO, "    NN        : 0x%x\n", Private->ControllerData->Nn));
 985
 986   return Status;
 987 }
 988