]> git.proxmox.com Git - mirror_edk2.git/blob - MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c
projects / mirror_edk2.git / blob
? search:


d87212ffb268ad12cc8a4a039b691001ed126928
[mirror_edk2.git] / MdeModulePkg / Bus / Pci / NvmExpressDxe / NvmExpressHci.c
1 /** @file
2 NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
3 NVM Express specification.
4
5 Copyright (c) 2013 - 2019, Intel Corporation. All rights reserved.<BR>
6 SPDX-License-Identifier: BSD-2-Clause-Patent
7
8 **/
9
10 #include "NvmExpress.h"
11
12 #define NVME_SHUTDOWN_PROCESS_TIMEOUT 45
13
14 //
15 // The number of NVME controllers managed by this driver, used by
16 // NvmeRegisterShutdownNotification() and NvmeUnregisterShutdownNotification().
17 //
18 UINTN mNvmeControllerNumber = 0;
19
20 /**
21 Read Nvm Express controller capability register.
22
23 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
24 @param Cap The buffer used to store capability register content.
25
26 @return EFI_SUCCESS Successfully read the controller capability register content.
27 @return EFI_DEVICE_ERROR Fail to read the controller capability register.
28
29 **/
30 EFI_STATUS
31 ReadNvmeControllerCapabilities (
32 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
33 IN NVME_CAP *Cap
34 )
35 {
36 EFI_PCI_IO_PROTOCOL *PciIo;
37 EFI_STATUS Status;
38 UINT64 Data;
39
40 PciIo = Private->PciIo;
41 Status = PciIo->Mem.Read (
42 PciIo,
43 EfiPciIoWidthUint32,
44 NVME_BAR,
45 NVME_CAP_OFFSET,
46 2,
47 &Data
48 );
49
50 if (EFI_ERROR (Status)) {
51 return Status;
52 }
53
54 WriteUnaligned64 ((UINT64 *)Cap, Data);
55 return EFI_SUCCESS;
56 }
57
58 /**
59 Read Nvm Express controller configuration register.
60
61 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
62 @param Cc The buffer used to store configuration register content.
63
64 @return EFI_SUCCESS Successfully read the controller configuration register content.
65 @return EFI_DEVICE_ERROR Fail to read the controller configuration register.
66
67 **/
68 EFI_STATUS
69 ReadNvmeControllerConfiguration (
70 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
71 IN NVME_CC *Cc
72 )
73 {
74 EFI_PCI_IO_PROTOCOL *PciIo;
75 EFI_STATUS Status;
76 UINT32 Data;
77
78 PciIo = Private->PciIo;
79 Status = PciIo->Mem.Read (
80 PciIo,
81 EfiPciIoWidthUint32,
82 NVME_BAR,
83 NVME_CC_OFFSET,
84 1,
85 &Data
86 );
87
88 if (EFI_ERROR (Status)) {
89 return Status;
90 }
91
92 WriteUnaligned32 ((UINT32 *)Cc, Data);
93 return EFI_SUCCESS;
94 }
95
96 /**
97 Write Nvm Express controller configuration register.
98
99 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
100 @param Cc The buffer used to store the content to be written into configuration register.
101
102 @return EFI_SUCCESS Successfully write data into the controller configuration register.
103 @return EFI_DEVICE_ERROR Fail to write data into the controller configuration register.
104
105 **/
106 EFI_STATUS
107 WriteNvmeControllerConfiguration (
108 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
109 IN NVME_CC *Cc
110 )
111 {
112 EFI_PCI_IO_PROTOCOL *PciIo;
113 EFI_STATUS Status;
114 UINT32 Data;
115
116 PciIo = Private->PciIo;
117 Data = ReadUnaligned32 ((UINT32 *)Cc);
118 Status = PciIo->Mem.Write (
119 PciIo,
120 EfiPciIoWidthUint32,
121 NVME_BAR,
122 NVME_CC_OFFSET,
123 1,
124 &Data
125 );
126
127 if (EFI_ERROR (Status)) {
128 return Status;
129 }
130
131 DEBUG ((DEBUG_INFO, "Cc.En: %d\n", Cc->En));
132 DEBUG ((DEBUG_INFO, "Cc.Css: %d\n", Cc->Css));
133 DEBUG ((DEBUG_INFO, "Cc.Mps: %d\n", Cc->Mps));
134 DEBUG ((DEBUG_INFO, "Cc.Ams: %d\n", Cc->Ams));
135 DEBUG ((DEBUG_INFO, "Cc.Shn: %d\n", Cc->Shn));
136 DEBUG ((DEBUG_INFO, "Cc.Iosqes: %d\n", Cc->Iosqes));
137 DEBUG ((DEBUG_INFO, "Cc.Iocqes: %d\n", Cc->Iocqes));
138
139 return EFI_SUCCESS;
140 }
141
142 /**
143 Read Nvm Express controller status register.
144
145 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
146 @param Csts The buffer used to store status register content.
147
148 @return EFI_SUCCESS Successfully read the controller status register content.
149 @return EFI_DEVICE_ERROR Fail to read the controller status register.
150
151 **/
152 EFI_STATUS
153 ReadNvmeControllerStatus (
154 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
155 IN NVME_CSTS *Csts
156 )
157 {
158 EFI_PCI_IO_PROTOCOL *PciIo;
159 EFI_STATUS Status;
160 UINT32 Data;
161
162 PciIo = Private->PciIo;
163 Status = PciIo->Mem.Read (
164 PciIo,
165 EfiPciIoWidthUint32,
166 NVME_BAR,
167 NVME_CSTS_OFFSET,
168 1,
169 &Data
170 );
171
172 if (EFI_ERROR (Status)) {
173 return Status;
174 }
175
176 WriteUnaligned32 ((UINT32 *)Csts, Data);
177 return EFI_SUCCESS;
178 }
179
180 /**
181 Write Nvm Express admin queue attributes register.
182
183 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
184 @param Aqa The buffer used to store the content to be written into admin queue attributes register.
185
186 @return EFI_SUCCESS Successfully write data into the admin queue attributes register.
187 @return EFI_DEVICE_ERROR Fail to write data into the admin queue attributes register.
188
189 **/
190 EFI_STATUS
191 WriteNvmeAdminQueueAttributes (
192 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
193 IN NVME_AQA *Aqa
194 )
195 {
196 EFI_PCI_IO_PROTOCOL *PciIo;
197 EFI_STATUS Status;
198 UINT32 Data;
199
200 PciIo = Private->PciIo;
201 Data = ReadUnaligned32 ((UINT32 *)Aqa);
202 Status = PciIo->Mem.Write (
203 PciIo,
204 EfiPciIoWidthUint32,
205 NVME_BAR,
206 NVME_AQA_OFFSET,
207 1,
208 &Data
209 );
210
211 if (EFI_ERROR (Status)) {
212 return Status;
213 }
214
215 DEBUG ((DEBUG_INFO, "Aqa.Asqs: %d\n", Aqa->Asqs));
216 DEBUG ((DEBUG_INFO, "Aqa.Acqs: %d\n", Aqa->Acqs));
217
218 return EFI_SUCCESS;
219 }
220
221 /**
222 Write Nvm Express admin submission queue base address register.
223
224 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
225 @param Asq The buffer used to store the content to be written into admin submission queue base address register.
226
227 @return EFI_SUCCESS Successfully write data into the admin submission queue base address register.
228 @return EFI_DEVICE_ERROR Fail to write data into the admin submission queue base address register.
229
230 **/
231 EFI_STATUS
232 WriteNvmeAdminSubmissionQueueBaseAddress (
233 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
234 IN NVME_ASQ *Asq
235 )
236 {
237 EFI_PCI_IO_PROTOCOL *PciIo;
238 EFI_STATUS Status;
239 UINT64 Data;
240
241 PciIo = Private->PciIo;
242 Data = ReadUnaligned64 ((UINT64 *)Asq);
243
244 Status = PciIo->Mem.Write (
245 PciIo,
246 EfiPciIoWidthUint32,
247 NVME_BAR,
248 NVME_ASQ_OFFSET,
249 2,
250 &Data
251 );
252
253 if (EFI_ERROR (Status)) {
254 return Status;
255 }
256
257 DEBUG ((DEBUG_INFO, "Asq: %lx\n", *Asq));
258
259 return EFI_SUCCESS;
260 }
261
262 /**
263 Write Nvm Express admin completion queue base address register.
264
265 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
266 @param Acq The buffer used to store the content to be written into admin completion queue base address register.
267
268 @return EFI_SUCCESS Successfully write data into the admin completion queue base address register.
269 @return EFI_DEVICE_ERROR Fail to write data into the admin completion queue base address register.
270
271 **/
272 EFI_STATUS
273 WriteNvmeAdminCompletionQueueBaseAddress (
274 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
275 IN NVME_ACQ *Acq
276 )
277 {
278 EFI_PCI_IO_PROTOCOL *PciIo;
279 EFI_STATUS Status;
280 UINT64 Data;
281
282 PciIo = Private->PciIo;
283 Data = ReadUnaligned64 ((UINT64 *)Acq);
284
285 Status = PciIo->Mem.Write (
286 PciIo,
287 EfiPciIoWidthUint32,
288 NVME_BAR,
289 NVME_ACQ_OFFSET,
290 2,
291 &Data
292 );
293
294 if (EFI_ERROR (Status)) {
295 return Status;
296 }
297
298 DEBUG ((DEBUG_INFO, "Acq: %lxh\n", *Acq));
299
300 return EFI_SUCCESS;
301 }
302
303 /**
304 Disable the Nvm Express controller.
305
306 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
307
308 @return EFI_SUCCESS Successfully disable the controller.
309 @return EFI_DEVICE_ERROR Fail to disable the controller.
310
311 **/
312 EFI_STATUS
313 NvmeDisableController (
314 IN NVME_CONTROLLER_PRIVATE_DATA *Private
315 )
316 {
317 NVME_CC Cc;
318 NVME_CSTS Csts;
319 EFI_STATUS Status;
320 UINT32 Index;
321 UINT8 Timeout;
322
323 //
324 // Read Controller Configuration Register.
325 //
326 Status = ReadNvmeControllerConfiguration (Private, &Cc);
327 if (EFI_ERROR (Status)) {
328 return Status;
329 }
330
331 Cc.En = 0;
332
333 //
334 // Disable the controller.
335 //
336 Status = WriteNvmeControllerConfiguration (Private, &Cc);
337
338 if (EFI_ERROR (Status)) {
339 return Status;
340 }
341
342 //
343 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to transition from 1 to 0 after
344 // Cc.Enable transition from 1 to 0. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
345 //
346 if (Private->Cap.To == 0) {
347 Timeout = 1;
348 } else {
349 Timeout = Private->Cap.To;
350 }
351
352 for (Index = (Timeout * 500); Index != 0; --Index) {
353 gBS->Stall (1000);
354
355 //
356 // Check if the controller is initialized
357 //
358 Status = ReadNvmeControllerStatus (Private, &Csts);
359
360 if (EFI_ERROR (Status)) {
361 return Status;
362 }
363
364 if (Csts.Rdy == 0) {
365 break;
366 }
367 }
368
369 if (Index == 0) {
370 Status = EFI_DEVICE_ERROR;
371 REPORT_STATUS_CODE (
372 (EFI_ERROR_CODE | EFI_ERROR_MAJOR),
373 (EFI_IO_BUS_SCSI | EFI_IOB_EC_INTERFACE_ERROR)
374 );
375 }
376
377 DEBUG ((DEBUG_INFO, "NVMe controller is disabled with status [%r].\n", Status));
378 return Status;
379 }
380
381 /**
382 Enable the Nvm Express controller.
383
384 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
385
386 @return EFI_SUCCESS Successfully enable the controller.
387 @return EFI_DEVICE_ERROR Fail to enable the controller.
388 @return EFI_TIMEOUT Fail to enable the controller in given time slot.
389
390 **/
391 EFI_STATUS
392 NvmeEnableController (
393 IN NVME_CONTROLLER_PRIVATE_DATA *Private
394 )
395 {
396 NVME_CC Cc;
397 NVME_CSTS Csts;
398 EFI_STATUS Status;
399 UINT32 Index;
400 UINT8 Timeout;
401
402 //
403 // Enable the controller.
404 // CC.AMS, CC.MPS and CC.CSS are all set to 0.
405 //
406 ZeroMem (&Cc, sizeof (NVME_CC));
407 Cc.En = 1;
408 Cc.Iosqes = 6;
409 Cc.Iocqes = 4;
410
411 Status = WriteNvmeControllerConfiguration (Private, &Cc);
412 if (EFI_ERROR (Status)) {
413 return Status;
414 }
415
416 //
417 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
418 // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
419 //
420 if (Private->Cap.To == 0) {
421 Timeout = 1;
422 } else {
423 Timeout = Private->Cap.To;
424 }
425
426 for (Index = (Timeout * 500); Index != 0; --Index) {
427 gBS->Stall (1000);
428
429 //
430 // Check if the controller is initialized
431 //
432 Status = ReadNvmeControllerStatus (Private, &Csts);
433
434 if (EFI_ERROR (Status)) {
435 return Status;
436 }
437
438 if (Csts.Rdy) {
439 break;
440 }
441 }
442
443 if (Index == 0) {
444 Status = EFI_TIMEOUT;
445 REPORT_STATUS_CODE (
446 (EFI_ERROR_CODE | EFI_ERROR_MAJOR),
447 (EFI_IO_BUS_SCSI | EFI_IOB_EC_INTERFACE_ERROR)
448 );
449 }
450
451 DEBUG ((DEBUG_INFO, "NVMe controller is enabled with status [%r].\n", Status));
452 return Status;
453 }
454
455 /**
456 Get identify controller data.
457
458 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
459 @param Buffer The buffer used to store the identify controller data.
460
461 @return EFI_SUCCESS Successfully get the identify controller data.
462 @return EFI_DEVICE_ERROR Fail to get the identify controller data.
463
464 **/
465 EFI_STATUS
466 NvmeIdentifyController (
467 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
468 IN VOID *Buffer
469 )
470 {
471 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
472 EFI_NVM_EXPRESS_COMMAND Command;
473 EFI_NVM_EXPRESS_COMPLETION Completion;
474 EFI_STATUS Status;
475
476 ZeroMem (&CommandPacket, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
477 ZeroMem (&Command, sizeof (EFI_NVM_EXPRESS_COMMAND));
478 ZeroMem (&Completion, sizeof (EFI_NVM_EXPRESS_COMPLETION));
479
480 Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;
481 //
482 // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
483 // For the Identify command, the Namespace Identifier is only used for the Namespace data structure.
484 //
485 Command.Nsid = 0;
486
487 CommandPacket.NvmeCmd = &Command;
488 CommandPacket.NvmeCompletion = &Completion;
489 CommandPacket.TransferBuffer = Buffer;
490 CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA);
491 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
492 CommandPacket.QueueType = NVME_ADMIN_QUEUE;
493 //
494 // Set bit 0 (Cns bit) to 1 to identify a controller
495 //
496 Command.Cdw10 = 1;
497 Command.Flags = CDW10_VALID;
498
499 Status = Private->Passthru.PassThru (
500 &Private->Passthru,
501 NVME_CONTROLLER_ID,
502 &CommandPacket,
503 NULL
504 );
505
506 return Status;
507 }
508
509 /**
510 Get specified identify namespace data.
511
512 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
513 @param NamespaceId The specified namespace identifier.
514 @param Buffer The buffer used to store the identify namespace data.
515
516 @return EFI_SUCCESS Successfully get the identify namespace data.
517 @return EFI_DEVICE_ERROR Fail to get the identify namespace data.
518
519 **/
520 EFI_STATUS
521 NvmeIdentifyNamespace (
522 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
523 IN UINT32 NamespaceId,
524 IN VOID *Buffer
525 )
526 {
527 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
528 EFI_NVM_EXPRESS_COMMAND Command;
529 EFI_NVM_EXPRESS_COMPLETION Completion;
530 EFI_STATUS Status;
531
532 ZeroMem (&CommandPacket, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
533 ZeroMem (&Command, sizeof (EFI_NVM_EXPRESS_COMMAND));
534 ZeroMem (&Completion, sizeof (EFI_NVM_EXPRESS_COMPLETION));
535
536 CommandPacket.NvmeCmd = &Command;
537 CommandPacket.NvmeCompletion = &Completion;
538
539 Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;
540 Command.Nsid = NamespaceId;
541 CommandPacket.TransferBuffer = Buffer;
542 CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA);
543 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
544 CommandPacket.QueueType = NVME_ADMIN_QUEUE;
545 //
546 // Set bit 0 (Cns bit) to 1 to identify a namespace
547 //
548 CommandPacket.NvmeCmd->Cdw10 = 0;
549 CommandPacket.NvmeCmd->Flags = CDW10_VALID;
550
551 Status = Private->Passthru.PassThru (
552 &Private->Passthru,
553 NamespaceId,
554 &CommandPacket,
555 NULL
556 );
557
558 return Status;
559 }
560
561 /**
562 Create io completion queue.
563
564 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
565
566 @return EFI_SUCCESS Successfully create io completion queue.
567 @return EFI_DEVICE_ERROR Fail to create io completion queue.
568
569 **/
570 EFI_STATUS
571 NvmeCreateIoCompletionQueue (
572 IN NVME_CONTROLLER_PRIVATE_DATA *Private
573 )
574 {
575 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
576 EFI_NVM_EXPRESS_COMMAND Command;
577 EFI_NVM_EXPRESS_COMPLETION Completion;
578 EFI_STATUS Status;
579 NVME_ADMIN_CRIOCQ CrIoCq;
580 UINT32 Index;
581 UINT16 QueueSize;
582
583 Status = EFI_SUCCESS;
584 Private->CreateIoQueue = TRUE;
585
586 for (Index = 1; Index < NVME_MAX_QUEUES; Index++) {
587 ZeroMem (&CommandPacket, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
588 ZeroMem (&Command, sizeof (EFI_NVM_EXPRESS_COMMAND));
589 ZeroMem (&Completion, sizeof (EFI_NVM_EXPRESS_COMPLETION));
590 ZeroMem (&CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
591
592 CommandPacket.NvmeCmd = &Command;
593 CommandPacket.NvmeCompletion = &Completion;
594
595 Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_CMD;
596 CommandPacket.TransferBuffer = Private->CqBufferPciAddr[Index];
597 CommandPacket.TransferLength = EFI_PAGE_SIZE;
598 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
599 CommandPacket.QueueType = NVME_ADMIN_QUEUE;
600
601 if (Index == 1) {
602 QueueSize = NVME_CCQ_SIZE;
603 } else {
604 if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) {
605 QueueSize = NVME_ASYNC_CCQ_SIZE;
606 } else {
607 QueueSize = Private->Cap.Mqes;
608 }
609 }
610
611 CrIoCq.Qid = Index;
612 CrIoCq.Qsize = QueueSize;
613 CrIoCq.Pc = 1;
614 CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
615 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
616
617 Status = Private->Passthru.PassThru (
618 &Private->Passthru,
619 0,
620 &CommandPacket,
621 NULL
622 );
623 if (EFI_ERROR (Status)) {
624 break;
625 }
626 }
627
628 Private->CreateIoQueue = FALSE;
629
630 return Status;
631 }
632
633 /**
634 Create io submission queue.
635
636 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
637
638 @return EFI_SUCCESS Successfully create io submission queue.
639 @return EFI_DEVICE_ERROR Fail to create io submission queue.
640
641 **/
642 EFI_STATUS
643 NvmeCreateIoSubmissionQueue (
644 IN NVME_CONTROLLER_PRIVATE_DATA *Private
645 )
646 {
647 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
648 EFI_NVM_EXPRESS_COMMAND Command;
649 EFI_NVM_EXPRESS_COMPLETION Completion;
650 EFI_STATUS Status;
651 NVME_ADMIN_CRIOSQ CrIoSq;
652 UINT32 Index;
653 UINT16 QueueSize;
654
655 Status = EFI_SUCCESS;
656 Private->CreateIoQueue = TRUE;
657
658 for (Index = 1; Index < NVME_MAX_QUEUES; Index++) {
659 ZeroMem (&CommandPacket, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
660 ZeroMem (&Command, sizeof (EFI_NVM_EXPRESS_COMMAND));
661 ZeroMem (&Completion, sizeof (EFI_NVM_EXPRESS_COMPLETION));
662 ZeroMem (&CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
663
664 CommandPacket.NvmeCmd = &Command;
665 CommandPacket.NvmeCompletion = &Completion;
666
667 Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_CMD;
668 CommandPacket.TransferBuffer = Private->SqBufferPciAddr[Index];
669 CommandPacket.TransferLength = EFI_PAGE_SIZE;
670 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
671 CommandPacket.QueueType = NVME_ADMIN_QUEUE;
672
673 if (Index == 1) {
674 QueueSize = NVME_CSQ_SIZE;
675 } else {
676 if (Private->Cap.Mqes > NVME_ASYNC_CSQ_SIZE) {
677 QueueSize = NVME_ASYNC_CSQ_SIZE;
678 } else {
679 QueueSize = Private->Cap.Mqes;
680 }
681 }
682
683 CrIoSq.Qid = Index;
684 CrIoSq.Qsize = QueueSize;
685 CrIoSq.Pc = 1;
686 CrIoSq.Cqid = Index;
687 CrIoSq.Qprio = 0;
688 CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
689 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
690
691 Status = Private->Passthru.PassThru (
692 &Private->Passthru,
693 0,
694 &CommandPacket,
695 NULL
696 );
697 if (EFI_ERROR (Status)) {
698 break;
699 }
700 }
701
702 Private->CreateIoQueue = FALSE;
703
704 return Status;
705 }
706
707 /**
708 Initialize the Nvm Express controller.
709
710 @param[in] Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
711
712 @retval EFI_SUCCESS The NVM Express Controller is initialized successfully.
713 @retval Others A device error occurred while initializing the controller.
714
715 **/
716 EFI_STATUS
717 NvmeControllerInit (
718 IN NVME_CONTROLLER_PRIVATE_DATA *Private
719 )
720 {
721 EFI_STATUS Status;
722 EFI_PCI_IO_PROTOCOL *PciIo;
723 UINT64 Supports;
724 NVME_AQA Aqa;
725 NVME_ASQ Asq;
726 NVME_ACQ Acq;
727 UINT8 Sn[21];
728 UINT8 Mn[41];
729
730 //
731 // Enable this controller.
732 //
733 PciIo = Private->PciIo;
734 Status = PciIo->Attributes (
735 PciIo,
736 EfiPciIoAttributeOperationSupported,
737 0,
738 &Supports
739 );
740
741 if (!EFI_ERROR (Status)) {
742 Supports &= (UINT64)EFI_PCI_DEVICE_ENABLE;
743 Status = PciIo->Attributes (
744 PciIo,
745 EfiPciIoAttributeOperationEnable,
746 Supports,
747 NULL
748 );
749 }
750
751 if (EFI_ERROR (Status)) {
752 DEBUG ((DEBUG_INFO, "NvmeControllerInit: failed to enable controller\n"));
753 return Status;
754 }
755
756 //
757 // Read the Controller Capabilities register and verify that the NVM command set is supported
758 //
759 Status = ReadNvmeControllerCapabilities (Private, &Private->Cap);
760 if (EFI_ERROR (Status)) {
761 return Status;
762 }
763
764 if (Private->Cap.Css != 0x01) {
765 DEBUG ((DEBUG_INFO, "NvmeControllerInit: the controller doesn't support NVMe command set\n"));
766 return EFI_UNSUPPORTED;
767 }
768
769 //
770 // Currently the driver only supports 4k page size.
771 //
772 ASSERT ((Private->Cap.Mpsmin + 12) <= EFI_PAGE_SHIFT);
773
774 Private->Cid[0] = 0;
775 Private->Cid[1] = 0;
776 Private->Cid[2] = 0;
777 Private->Pt[0] = 0;
778 Private->Pt[1] = 0;
779 Private->Pt[2] = 0;
780 Private->SqTdbl[0].Sqt = 0;
781 Private->SqTdbl[1].Sqt = 0;
782 Private->SqTdbl[2].Sqt = 0;
783 Private->CqHdbl[0].Cqh = 0;
784 Private->CqHdbl[1].Cqh = 0;
785 Private->CqHdbl[2].Cqh = 0;
786 Private->AsyncSqHead = 0;
787
788 Status = NvmeDisableController (Private);
789
790 if (EFI_ERROR (Status)) {
791 return Status;
792 }
793
794 //
795 // set number of entries admin submission & completion queues.
796 //
797 Aqa.Asqs = NVME_ASQ_SIZE;
798 Aqa.Rsvd1 = 0;
799 Aqa.Acqs = NVME_ACQ_SIZE;
800 Aqa.Rsvd2 = 0;
801
802 //
803 // Address of admin submission queue.
804 //
805 Asq = (UINT64)(UINTN)(Private->BufferPciAddr) & ~0xFFF;
806
807 //
808 // Address of admin completion queue.
809 //
810 Acq = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) & ~0xFFF;
811
812 //
813 // Address of I/O submission & completion queue.
814 //
815 ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (6));
816 Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer);
817 Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr);
818 Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE);
819 Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 1 * EFI_PAGE_SIZE);
820 Private->SqBuffer[1] = (NVME_SQ *)(UINTN)(Private->Buffer + 2 * EFI_PAGE_SIZE);
821 Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE);
822 Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE);
823 Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE);
824 Private->SqBuffer[2] = (NVME_SQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE);
825 Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE);
826 Private->CqBuffer[2] = (NVME_CQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE);
827 Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE);
828
829 DEBUG ((DEBUG_INFO, "Private->Buffer = [%016X]\n", (UINT64)(UINTN)Private->Buffer));
830 DEBUG ((DEBUG_INFO, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));
831 DEBUG ((DEBUG_INFO, "Admin Completion Queue size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));
832 DEBUG ((DEBUG_INFO, "Admin Submission Queue (SqBuffer[0]) = [%016X]\n", Private->SqBuffer[0]));
833 DEBUG ((DEBUG_INFO, "Admin Completion Queue (CqBuffer[0]) = [%016X]\n", Private->CqBuffer[0]));
834 DEBUG ((DEBUG_INFO, "Sync I/O Submission Queue (SqBuffer[1]) = [%016X]\n", Private->SqBuffer[1]));
835 DEBUG ((DEBUG_INFO, "Sync I/O Completion Queue (CqBuffer[1]) = [%016X]\n", Private->CqBuffer[1]));
836 DEBUG ((DEBUG_INFO, "Async I/O Submission Queue (SqBuffer[2]) = [%016X]\n", Private->SqBuffer[2]));
837 DEBUG ((DEBUG_INFO, "Async I/O Completion Queue (CqBuffer[2]) = [%016X]\n", Private->CqBuffer[2]));
838
839 //
840 // Program admin queue attributes.
841 //
842 Status = WriteNvmeAdminQueueAttributes (Private, &Aqa);
843
844 if (EFI_ERROR (Status)) {
845 return Status;
846 }
847
848 //
849 // Program admin submission queue address.
850 //
851 Status = WriteNvmeAdminSubmissionQueueBaseAddress (Private, &Asq);
852
853 if (EFI_ERROR (Status)) {
854 return Status;
855 }
856
857 //
858 // Program admin completion queue address.
859 //
860 Status = WriteNvmeAdminCompletionQueueBaseAddress (Private, &Acq);
861
862 if (EFI_ERROR (Status)) {
863 return Status;
864 }
865
866 Status = NvmeEnableController (Private);
867 if (EFI_ERROR (Status)) {
868 return Status;
869 }
870
871 //
872 // Allocate buffer for Identify Controller data
873 //
874 if (Private->ControllerData == NULL) {
875 Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_CONTROLLER_DATA));
876
877 if (Private->ControllerData == NULL) {
878 return EFI_OUT_OF_RESOURCES;
879 }
880 }
881
882 //
883 // Get current Identify Controller Data
884 //
885 Status = NvmeIdentifyController (Private, Private->ControllerData);
886
887 if (EFI_ERROR (Status)) {
888 FreePool (Private->ControllerData);
889 Private->ControllerData = NULL;
890 return EFI_NOT_FOUND;
891 }
892
893 //
894 // Dump NvmExpress Identify Controller Data
895 //
896 CopyMem (Sn, Private->ControllerData->Sn, sizeof (Private->ControllerData->Sn));
897 Sn[20] = 0;
898 CopyMem (Mn, Private->ControllerData->Mn, sizeof (Private->ControllerData->Mn));
899 Mn[40] = 0;
900 DEBUG ((DEBUG_INFO, " == NVME IDENTIFY CONTROLLER DATA ==\n"));
901 DEBUG ((DEBUG_INFO, " PCI VID : 0x%x\n", Private->ControllerData->Vid));
902 DEBUG ((DEBUG_INFO, " PCI SSVID : 0x%x\n", Private->ControllerData->Ssvid));
903 DEBUG ((DEBUG_INFO, " SN : %a\n", Sn));
904 DEBUG ((DEBUG_INFO, " MN : %a\n", Mn));
905 DEBUG ((DEBUG_INFO, " FR : 0x%x\n", *((UINT64 *)Private->ControllerData->Fr)));
906 DEBUG ((DEBUG_INFO, " TNVMCAP (high 8-byte) : 0x%lx\n", *((UINT64 *)(Private->ControllerData->Tnvmcap + 8))));
907 DEBUG ((DEBUG_INFO, " TNVMCAP (low 8-byte) : 0x%lx\n", *((UINT64 *)Private->ControllerData->Tnvmcap)));
908 DEBUG ((DEBUG_INFO, " RAB : 0x%x\n", Private->ControllerData->Rab));
909 DEBUG ((DEBUG_INFO, " IEEE : 0x%x\n", *(UINT32 *)Private->ControllerData->Ieee_oui));
910 DEBUG ((DEBUG_INFO, " AERL : 0x%x\n", Private->ControllerData->Aerl));
911 DEBUG ((DEBUG_INFO, " SQES : 0x%x\n", Private->ControllerData->Sqes));
912 DEBUG ((DEBUG_INFO, " CQES : 0x%x\n", Private->ControllerData->Cqes));
913 DEBUG ((DEBUG_INFO, " NN : 0x%x\n", Private->ControllerData->Nn));
914
915 //
916 // Create two I/O completion queues.
917 // One for blocking I/O, one for non-blocking I/O.
918 //
919 Status = NvmeCreateIoCompletionQueue (Private);
920 if (EFI_ERROR (Status)) {
921 return Status;
922 }
923
924 //
925 // Create two I/O Submission queues.
926 // One for blocking I/O, one for non-blocking I/O.
927 //
928 Status = NvmeCreateIoSubmissionQueue (Private);
929
930 return Status;
931 }
932
933 /**
934 This routine is called to properly shutdown the Nvm Express controller per NVMe spec.
935
936 @param[in] ResetType The type of reset to perform.
937 @param[in] ResetStatus The status code for the reset.
938 @param[in] DataSize The size, in bytes, of ResetData.
939 @param[in] ResetData For a ResetType of EfiResetCold, EfiResetWarm, or
940 EfiResetShutdown the data buffer starts with a Null-terminated
941 string, optionally followed by additional binary data.
942 The string is a description that the caller may use to further
943 indicate the reason for the system reset.
944 For a ResetType of EfiResetPlatformSpecific the data buffer
945 also starts with a Null-terminated string that is followed
946 by an EFI_GUID that describes the specific type of reset to perform.
947 **/
948 VOID
949 EFIAPI
950 NvmeShutdownAllControllers (
951 IN EFI_RESET_TYPE ResetType,
952 IN EFI_STATUS ResetStatus,
953 IN UINTN DataSize,
954 IN VOID *ResetData OPTIONAL
955 )
956 {
957 EFI_STATUS Status;
958 EFI_HANDLE *Handles;
959 UINTN HandleCount;
960 UINTN HandleIndex;
961 EFI_OPEN_PROTOCOL_INFORMATION_ENTRY *OpenInfos;
962 UINTN OpenInfoCount;
963 UINTN OpenInfoIndex;
964 EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *NvmePassThru;
965 NVME_CC Cc;
966 NVME_CSTS Csts;
967 UINTN Index;
968 NVME_CONTROLLER_PRIVATE_DATA *Private;
969
970 Status = gBS->LocateHandleBuffer (
971 ByProtocol,
972 &gEfiPciIoProtocolGuid,
973 NULL,
974 &HandleCount,
975 &Handles
976 );
977 if (EFI_ERROR (Status)) {
978 HandleCount = 0;
979 }
980
981 for (HandleIndex = 0; HandleIndex < HandleCount; HandleIndex++) {
982 Status = gBS->OpenProtocolInformation (
983 Handles[HandleIndex],
984 &gEfiPciIoProtocolGuid,
985 &OpenInfos,
986 &OpenInfoCount
987 );
988 if (EFI_ERROR (Status)) {
989 continue;
990 }
991
992 for (OpenInfoIndex = 0; OpenInfoIndex < OpenInfoCount; OpenInfoIndex++) {
993 //
994 // Find all the NVME controller managed by this driver.
995 // gImageHandle equals to DriverBinding handle for this driver.
996 //
997 if (((OpenInfos[OpenInfoIndex].Attributes & EFI_OPEN_PROTOCOL_BY_DRIVER) != 0) &&
998 (OpenInfos[OpenInfoIndex].AgentHandle == gImageHandle))
999 {
1000 Status = gBS->OpenProtocol (
1001 OpenInfos[OpenInfoIndex].ControllerHandle,
1002 &gEfiNvmExpressPassThruProtocolGuid,
1003 (VOID **)&NvmePassThru,
1004 NULL,
1005 NULL,
1006 EFI_OPEN_PROTOCOL_GET_PROTOCOL
1007 );
1008 if (EFI_ERROR (Status)) {
1009 continue;
1010 }
1011
1012 Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (NvmePassThru);
1013
1014 //
1015 // Read Controller Configuration Register.
1016 //
1017 Status = ReadNvmeControllerConfiguration (Private, &Cc);
1018 if (EFI_ERROR (Status)) {
1019 continue;
1020 }
1021
1022 //
1023 // The host should set the Shutdown Notification (CC.SHN) field to 01b
1024 // to indicate a normal shutdown operation.
1025 //
1026 Cc.Shn = NVME_CC_SHN_NORMAL_SHUTDOWN;
1027 Status = WriteNvmeControllerConfiguration (Private, &Cc);
1028 if (EFI_ERROR (Status)) {
1029 continue;
1030 }
1031
1032 //
1033 // The controller indicates when shutdown processing is completed by updating the
1034 // Shutdown Status (CSTS.SHST) field to 10b.
1035 // Wait up to 45 seconds (break down to 4500 x 10ms) for the shutdown to complete.
1036 //
1037 for (Index = 0; Index < NVME_SHUTDOWN_PROCESS_TIMEOUT * 100; Index++) {
1038 Status = ReadNvmeControllerStatus (Private, &Csts);
1039 if (!EFI_ERROR (Status) && (Csts.Shst == NVME_CSTS_SHST_SHUTDOWN_COMPLETED)) {
1040 DEBUG ((DEBUG_INFO, "NvmeShutdownController: shutdown processing is completed after %dms.\n", Index * 10));
1041 break;
1042 }
1043
1044 //
1045 // Stall for 10ms
1046 //
1047 gBS->Stall (10 * 1000);
1048 }
1049
1050 if (Index == NVME_SHUTDOWN_PROCESS_TIMEOUT * 100) {
1051 DEBUG ((DEBUG_ERROR, "NvmeShutdownController: shutdown processing is timed out\n"));
1052 }
1053 }
1054 }
1055 }
1056 }
1057
1058 /**
1059 Register the shutdown notification through the ResetNotification protocol.
1060
1061 Register the shutdown notification when mNvmeControllerNumber increased from 0 to 1.
1062 **/
1063 VOID
1064 NvmeRegisterShutdownNotification (
1065 VOID
1066 )
1067 {
1068 EFI_STATUS Status;
1069 EFI_RESET_NOTIFICATION_PROTOCOL *ResetNotify;
1070
1071 mNvmeControllerNumber++;
1072 if (mNvmeControllerNumber == 1) {
1073 Status = gBS->LocateProtocol (&gEfiResetNotificationProtocolGuid, NULL, (VOID **)&ResetNotify);
1074 if (!EFI_ERROR (Status)) {
1075 Status = ResetNotify->RegisterResetNotify (ResetNotify, NvmeShutdownAllControllers);
1076 ASSERT_EFI_ERROR (Status);
1077 } else {
1078 DEBUG ((DEBUG_WARN, "NVME: ResetNotification absent! Shutdown notification cannot be performed!\n"));
1079 }
1080 }
1081 }
1082
1083 /**
1084 Unregister the shutdown notification through the ResetNotification protocol.
1085
1086 Unregister the shutdown notification when mNvmeControllerNumber decreased from 1 to 0.
1087 **/
1088 VOID
1089 NvmeUnregisterShutdownNotification (
1090 VOID
1091 )
1092 {
1093 EFI_STATUS Status;
1094 EFI_RESET_NOTIFICATION_PROTOCOL *ResetNotify;
1095
1096 mNvmeControllerNumber--;
1097 if (mNvmeControllerNumber == 0) {
1098 Status = gBS->LocateProtocol (&gEfiResetNotificationProtocolGuid, NULL, (VOID **)&ResetNotify);
1099 if (!EFI_ERROR (Status)) {
1100 Status = ResetNotify->UnregisterResetNotify (ResetNotify, NvmeShutdownAllControllers);
1101 ASSERT_EFI_ERROR (Status);
1102 }
1103 }
1104 }
EFI Development Kit II mirror for PVE