]> git.proxmox.com Git - mirror_edk2.git/blob - MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c
projects / mirror_edk2.git / blob
? search:


4a070f3f13367e669e1cb6eb3a8c471490ca7f7e
[mirror_edk2.git] / MdeModulePkg / Bus / Pci / NvmExpressDxe / NvmExpressHci.c
1 /** @file
2 NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
3 NVM Express specification.
4
5 Copyright (c) 2013 - 2018, Intel Corporation. All rights reserved.<BR>
6 This program and the accompanying materials
7 are licensed and made available under the terms and conditions of the BSD License
8 which accompanies this distribution. The full text of the license may be found at
9 http://opensource.org/licenses/bsd-license.php.
10
11 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
12 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
13
14 **/
15
16 #include "NvmExpress.h"
17
18 #define NVME_SHUTDOWN_PROCESS_TIMEOUT 45
19
20 //
21 // The number of NVME controllers managed by this driver, used by
22 // NvmeRegisterShutdownNotification() and NvmeUnregisterShutdownNotification().
23 //
24 UINTN mNvmeControllerNumber = 0;
25
26 /**
27 Read Nvm Express controller capability register.
28
29 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
30 @param Cap The buffer used to store capability register content.
31
32 @return EFI_SUCCESS Successfully read the controller capability register content.
33 @return EFI_DEVICE_ERROR Fail to read the controller capability register.
34
35 **/
36 EFI_STATUS
37 ReadNvmeControllerCapabilities (
38 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
39 IN NVME_CAP *Cap
40 )
41 {
42 EFI_PCI_IO_PROTOCOL *PciIo;
43 EFI_STATUS Status;
44 UINT64 Data;
45
46 PciIo = Private->PciIo;
47 Status = PciIo->Mem.Read (
48 PciIo,
49 EfiPciIoWidthUint32,
50 NVME_BAR,
51 NVME_CAP_OFFSET,
52 2,
53 &Data
54 );
55
56 if (EFI_ERROR(Status)) {
57 return Status;
58 }
59
60 WriteUnaligned64 ((UINT64*)Cap, Data);
61 return EFI_SUCCESS;
62 }
63
64 /**
65 Read Nvm Express controller configuration register.
66
67 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
68 @param Cc The buffer used to store configuration register content.
69
70 @return EFI_SUCCESS Successfully read the controller configuration register content.
71 @return EFI_DEVICE_ERROR Fail to read the controller configuration register.
72
73 **/
74 EFI_STATUS
75 ReadNvmeControllerConfiguration (
76 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
77 IN NVME_CC *Cc
78 )
79 {
80 EFI_PCI_IO_PROTOCOL *PciIo;
81 EFI_STATUS Status;
82 UINT32 Data;
83
84 PciIo = Private->PciIo;
85 Status = PciIo->Mem.Read (
86 PciIo,
87 EfiPciIoWidthUint32,
88 NVME_BAR,
89 NVME_CC_OFFSET,
90 1,
91 &Data
92 );
93
94 if (EFI_ERROR(Status)) {
95 return Status;
96 }
97
98 WriteUnaligned32 ((UINT32*)Cc, Data);
99 return EFI_SUCCESS;
100 }
101
102 /**
103 Write Nvm Express controller configuration register.
104
105 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
106 @param Cc The buffer used to store the content to be written into configuration register.
107
108 @return EFI_SUCCESS Successfully write data into the controller configuration register.
109 @return EFI_DEVICE_ERROR Fail to write data into the controller configuration register.
110
111 **/
112 EFI_STATUS
113 WriteNvmeControllerConfiguration (
114 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
115 IN NVME_CC *Cc
116 )
117 {
118 EFI_PCI_IO_PROTOCOL *PciIo;
119 EFI_STATUS Status;
120 UINT32 Data;
121
122 PciIo = Private->PciIo;
123 Data = ReadUnaligned32 ((UINT32*)Cc);
124 Status = PciIo->Mem.Write (
125 PciIo,
126 EfiPciIoWidthUint32,
127 NVME_BAR,
128 NVME_CC_OFFSET,
129 1,
130 &Data
131 );
132
133 if (EFI_ERROR(Status)) {
134 return Status;
135 }
136
137 DEBUG ((EFI_D_INFO, "Cc.En: %d\n", Cc->En));
138 DEBUG ((EFI_D_INFO, "Cc.Css: %d\n", Cc->Css));
139 DEBUG ((EFI_D_INFO, "Cc.Mps: %d\n", Cc->Mps));
140 DEBUG ((EFI_D_INFO, "Cc.Ams: %d\n", Cc->Ams));
141 DEBUG ((EFI_D_INFO, "Cc.Shn: %d\n", Cc->Shn));
142 DEBUG ((EFI_D_INFO, "Cc.Iosqes: %d\n", Cc->Iosqes));
143 DEBUG ((EFI_D_INFO, "Cc.Iocqes: %d\n", Cc->Iocqes));
144
145 return EFI_SUCCESS;
146 }
147
148 /**
149 Read Nvm Express controller status register.
150
151 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
152 @param Csts The buffer used to store status register content.
153
154 @return EFI_SUCCESS Successfully read the controller status register content.
155 @return EFI_DEVICE_ERROR Fail to read the controller status register.
156
157 **/
158 EFI_STATUS
159 ReadNvmeControllerStatus (
160 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
161 IN NVME_CSTS *Csts
162 )
163 {
164 EFI_PCI_IO_PROTOCOL *PciIo;
165 EFI_STATUS Status;
166 UINT32 Data;
167
168 PciIo = Private->PciIo;
169 Status = PciIo->Mem.Read (
170 PciIo,
171 EfiPciIoWidthUint32,
172 NVME_BAR,
173 NVME_CSTS_OFFSET,
174 1,
175 &Data
176 );
177
178 if (EFI_ERROR(Status)) {
179 return Status;
180 }
181
182 WriteUnaligned32 ((UINT32*)Csts, Data);
183 return EFI_SUCCESS;
184 }
185
186
187
188 /**
189 Write Nvm Express admin queue attributes register.
190
191 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
192 @param Aqa The buffer used to store the content to be written into admin queue attributes register.
193
194 @return EFI_SUCCESS Successfully write data into the admin queue attributes register.
195 @return EFI_DEVICE_ERROR Fail to write data into the admin queue attributes register.
196
197 **/
198 EFI_STATUS
199 WriteNvmeAdminQueueAttributes (
200 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
201 IN NVME_AQA *Aqa
202 )
203 {
204 EFI_PCI_IO_PROTOCOL *PciIo;
205 EFI_STATUS Status;
206 UINT32 Data;
207
208 PciIo = Private->PciIo;
209 Data = ReadUnaligned32 ((UINT32*)Aqa);
210 Status = PciIo->Mem.Write (
211 PciIo,
212 EfiPciIoWidthUint32,
213 NVME_BAR,
214 NVME_AQA_OFFSET,
215 1,
216 &Data
217 );
218
219 if (EFI_ERROR(Status)) {
220 return Status;
221 }
222
223 DEBUG ((EFI_D_INFO, "Aqa.Asqs: %d\n", Aqa->Asqs));
224 DEBUG ((EFI_D_INFO, "Aqa.Acqs: %d\n", Aqa->Acqs));
225
226 return EFI_SUCCESS;
227 }
228
229
230 /**
231 Write Nvm Express admin submission queue base address register.
232
233 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
234 @param Asq The buffer used to store the content to be written into admin submission queue base address register.
235
236 @return EFI_SUCCESS Successfully write data into the admin submission queue base address register.
237 @return EFI_DEVICE_ERROR Fail to write data into the admin submission queue base address register.
238
239 **/
240 EFI_STATUS
241 WriteNvmeAdminSubmissionQueueBaseAddress (
242 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
243 IN NVME_ASQ *Asq
244 )
245 {
246 EFI_PCI_IO_PROTOCOL *PciIo;
247 EFI_STATUS Status;
248 UINT64 Data;
249
250 PciIo = Private->PciIo;
251 Data = ReadUnaligned64 ((UINT64*)Asq);
252
253 Status = PciIo->Mem.Write (
254 PciIo,
255 EfiPciIoWidthUint32,
256 NVME_BAR,
257 NVME_ASQ_OFFSET,
258 2,
259 &Data
260 );
261
262 if (EFI_ERROR(Status)) {
263 return Status;
264 }
265
266 DEBUG ((EFI_D_INFO, "Asq: %lx\n", *Asq));
267
268 return EFI_SUCCESS;
269 }
270
271
272
273 /**
274 Write Nvm Express admin completion queue base address register.
275
276 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
277 @param Acq The buffer used to store the content to be written into admin completion queue base address register.
278
279 @return EFI_SUCCESS Successfully write data into the admin completion queue base address register.
280 @return EFI_DEVICE_ERROR Fail to write data into the admin completion queue base address register.
281
282 **/
283 EFI_STATUS
284 WriteNvmeAdminCompletionQueueBaseAddress (
285 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
286 IN NVME_ACQ *Acq
287 )
288 {
289 EFI_PCI_IO_PROTOCOL *PciIo;
290 EFI_STATUS Status;
291 UINT64 Data;
292
293 PciIo = Private->PciIo;
294 Data = ReadUnaligned64 ((UINT64*)Acq);
295
296 Status = PciIo->Mem.Write (
297 PciIo,
298 EfiPciIoWidthUint32,
299 NVME_BAR,
300 NVME_ACQ_OFFSET,
301 2,
302 &Data
303 );
304
305 if (EFI_ERROR(Status)) {
306 return Status;
307 }
308
309 DEBUG ((EFI_D_INFO, "Acq: %lxh\n", *Acq));
310
311 return EFI_SUCCESS;
312 }
313
314 /**
315 Disable the Nvm Express controller.
316
317 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
318
319 @return EFI_SUCCESS Successfully disable the controller.
320 @return EFI_DEVICE_ERROR Fail to disable the controller.
321
322 **/
323 EFI_STATUS
324 NvmeDisableController (
325 IN NVME_CONTROLLER_PRIVATE_DATA *Private
326 )
327 {
328 NVME_CC Cc;
329 NVME_CSTS Csts;
330 EFI_STATUS Status;
331 UINT32 Index;
332 UINT8 Timeout;
333
334 //
335 // Read Controller Configuration Register.
336 //
337 Status = ReadNvmeControllerConfiguration (Private, &Cc);
338 if (EFI_ERROR(Status)) {
339 return Status;
340 }
341
342 Cc.En = 0;
343
344 //
345 // Disable the controller.
346 //
347 Status = WriteNvmeControllerConfiguration (Private, &Cc);
348
349 if (EFI_ERROR(Status)) {
350 return Status;
351 }
352
353 //
354 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to transition from 1 to 0 after
355 // Cc.Enable transition from 1 to 0. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
356 //
357 if (Private->Cap.To == 0) {
358 Timeout = 1;
359 } else {
360 Timeout = Private->Cap.To;
361 }
362
363 for(Index = (Timeout * 500); Index != 0; --Index) {
364 gBS->Stall(1000);
365
366 //
367 // Check if the controller is initialized
368 //
369 Status = ReadNvmeControllerStatus (Private, &Csts);
370
371 if (EFI_ERROR(Status)) {
372 return Status;
373 }
374
375 if (Csts.Rdy == 0) {
376 break;
377 }
378 }
379
380 if (Index == 0) {
381 Status = EFI_DEVICE_ERROR;
382 }
383
384 DEBUG ((EFI_D_INFO, "NVMe controller is disabled with status [%r].\n", Status));
385 return Status;
386 }
387
388 /**
389 Enable the Nvm Express controller.
390
391 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
392
393 @return EFI_SUCCESS Successfully enable the controller.
394 @return EFI_DEVICE_ERROR Fail to enable the controller.
395 @return EFI_TIMEOUT Fail to enable the controller in given time slot.
396
397 **/
398 EFI_STATUS
399 NvmeEnableController (
400 IN NVME_CONTROLLER_PRIVATE_DATA *Private
401 )
402 {
403 NVME_CC Cc;
404 NVME_CSTS Csts;
405 EFI_STATUS Status;
406 UINT32 Index;
407 UINT8 Timeout;
408
409 //
410 // Enable the controller.
411 // CC.AMS, CC.MPS and CC.CSS are all set to 0.
412 //
413 ZeroMem (&Cc, sizeof (NVME_CC));
414 Cc.En = 1;
415 Cc.Iosqes = 6;
416 Cc.Iocqes = 4;
417
418 Status = WriteNvmeControllerConfiguration (Private, &Cc);
419 if (EFI_ERROR(Status)) {
420 return Status;
421 }
422
423 //
424 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
425 // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
426 //
427 if (Private->Cap.To == 0) {
428 Timeout = 1;
429 } else {
430 Timeout = Private->Cap.To;
431 }
432
433 for(Index = (Timeout * 500); Index != 0; --Index) {
434 gBS->Stall(1000);
435
436 //
437 // Check if the controller is initialized
438 //
439 Status = ReadNvmeControllerStatus (Private, &Csts);
440
441 if (EFI_ERROR(Status)) {
442 return Status;
443 }
444
445 if (Csts.Rdy) {
446 break;
447 }
448 }
449
450 if (Index == 0) {
451 Status = EFI_TIMEOUT;
452 }
453
454 DEBUG ((EFI_D_INFO, "NVMe controller is enabled with status [%r].\n", Status));
455 return Status;
456 }
457
458 /**
459 Get identify controller data.
460
461 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
462 @param Buffer The buffer used to store the identify controller data.
463
464 @return EFI_SUCCESS Successfully get the identify controller data.
465 @return EFI_DEVICE_ERROR Fail to get the identify controller data.
466
467 **/
468 EFI_STATUS
469 NvmeIdentifyController (
470 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
471 IN VOID *Buffer
472 )
473 {
474 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
475 EFI_NVM_EXPRESS_COMMAND Command;
476 EFI_NVM_EXPRESS_COMPLETION Completion;
477 EFI_STATUS Status;
478
479 ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
480 ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));
481 ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));
482
483 Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;
484 //
485 // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
486 // For the Identify command, the Namespace Identifier is only used for the Namespace data structure.
487 //
488 Command.Nsid = 0;
489
490 CommandPacket.NvmeCmd = &Command;
491 CommandPacket.NvmeCompletion = &Completion;
492 CommandPacket.TransferBuffer = Buffer;
493 CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA);
494 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
495 CommandPacket.QueueType = NVME_ADMIN_QUEUE;
496 //
497 // Set bit 0 (Cns bit) to 1 to identify a controller
498 //
499 Command.Cdw10 = 1;
500 Command.Flags = CDW10_VALID;
501
502 Status = Private->Passthru.PassThru (
503 &Private->Passthru,
504 NVME_CONTROLLER_ID,
505 &CommandPacket,
506 NULL
507 );
508
509 return Status;
510 }
511
512 /**
513 Get specified identify namespace data.
514
515 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
516 @param NamespaceId The specified namespace identifier.
517 @param Buffer The buffer used to store the identify namespace data.
518
519 @return EFI_SUCCESS Successfully get the identify namespace data.
520 @return EFI_DEVICE_ERROR Fail to get the identify namespace data.
521
522 **/
523 EFI_STATUS
524 NvmeIdentifyNamespace (
525 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
526 IN UINT32 NamespaceId,
527 IN VOID *Buffer
528 )
529 {
530 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
531 EFI_NVM_EXPRESS_COMMAND Command;
532 EFI_NVM_EXPRESS_COMPLETION Completion;
533 EFI_STATUS Status;
534
535 ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
536 ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));
537 ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));
538
539 CommandPacket.NvmeCmd = &Command;
540 CommandPacket.NvmeCompletion = &Completion;
541
542 Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;
543 Command.Nsid = NamespaceId;
544 CommandPacket.TransferBuffer = Buffer;
545 CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA);
546 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
547 CommandPacket.QueueType = NVME_ADMIN_QUEUE;
548 //
549 // Set bit 0 (Cns bit) to 1 to identify a namespace
550 //
551 CommandPacket.NvmeCmd->Cdw10 = 0;
552 CommandPacket.NvmeCmd->Flags = CDW10_VALID;
553
554 Status = Private->Passthru.PassThru (
555 &Private->Passthru,
556 NamespaceId,
557 &CommandPacket,
558 NULL
559 );
560
561 return Status;
562 }
563
564 /**
565 Create io completion queue.
566
567 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
568
569 @return EFI_SUCCESS Successfully create io completion queue.
570 @return EFI_DEVICE_ERROR Fail to create io completion queue.
571
572 **/
573 EFI_STATUS
574 NvmeCreateIoCompletionQueue (
575 IN NVME_CONTROLLER_PRIVATE_DATA *Private
576 )
577 {
578 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
579 EFI_NVM_EXPRESS_COMMAND Command;
580 EFI_NVM_EXPRESS_COMPLETION Completion;
581 EFI_STATUS Status;
582 NVME_ADMIN_CRIOCQ CrIoCq;
583 UINT32 Index;
584 UINT16 QueueSize;
585
586 Status = EFI_SUCCESS;
587 Private->CreateIoQueue = TRUE;
588
589 for (Index = 1; Index < NVME_MAX_QUEUES; Index++) {
590 ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
591 ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));
592 ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));
593 ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ));
594
595 CommandPacket.NvmeCmd = &Command;
596 CommandPacket.NvmeCompletion = &Completion;
597
598 Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_CMD;
599 CommandPacket.TransferBuffer = Private->CqBufferPciAddr[Index];
600 CommandPacket.TransferLength = EFI_PAGE_SIZE;
601 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
602 CommandPacket.QueueType = NVME_ADMIN_QUEUE;
603
604 if (Index == 1) {
605 QueueSize = NVME_CCQ_SIZE;
606 } else {
607 if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) {
608 QueueSize = NVME_ASYNC_CCQ_SIZE;
609 } else {
610 QueueSize = Private->Cap.Mqes;
611 }
612 }
613
614 CrIoCq.Qid = Index;
615 CrIoCq.Qsize = QueueSize;
616 CrIoCq.Pc = 1;
617 CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
618 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
619
620 Status = Private->Passthru.PassThru (
621 &Private->Passthru,
622 0,
623 &CommandPacket,
624 NULL
625 );
626 if (EFI_ERROR (Status)) {
627 break;
628 }
629 }
630
631 Private->CreateIoQueue = FALSE;
632
633 return Status;
634 }
635
636 /**
637 Create io submission queue.
638
639 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
640
641 @return EFI_SUCCESS Successfully create io submission queue.
642 @return EFI_DEVICE_ERROR Fail to create io submission queue.
643
644 **/
645 EFI_STATUS
646 NvmeCreateIoSubmissionQueue (
647 IN NVME_CONTROLLER_PRIVATE_DATA *Private
648 )
649 {
650 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
651 EFI_NVM_EXPRESS_COMMAND Command;
652 EFI_NVM_EXPRESS_COMPLETION Completion;
653 EFI_STATUS Status;
654 NVME_ADMIN_CRIOSQ CrIoSq;
655 UINT32 Index;
656 UINT16 QueueSize;
657
658 Status = EFI_SUCCESS;
659 Private->CreateIoQueue = TRUE;
660
661 for (Index = 1; Index < NVME_MAX_QUEUES; Index++) {
662 ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
663 ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));
664 ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));
665 ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ));
666
667 CommandPacket.NvmeCmd = &Command;
668 CommandPacket.NvmeCompletion = &Completion;
669
670 Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_CMD;
671 CommandPacket.TransferBuffer = Private->SqBufferPciAddr[Index];
672 CommandPacket.TransferLength = EFI_PAGE_SIZE;
673 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
674 CommandPacket.QueueType = NVME_ADMIN_QUEUE;
675
676 if (Index == 1) {
677 QueueSize = NVME_CSQ_SIZE;
678 } else {
679 if (Private->Cap.Mqes > NVME_ASYNC_CSQ_SIZE) {
680 QueueSize = NVME_ASYNC_CSQ_SIZE;
681 } else {
682 QueueSize = Private->Cap.Mqes;
683 }
684 }
685
686 CrIoSq.Qid = Index;
687 CrIoSq.Qsize = QueueSize;
688 CrIoSq.Pc = 1;
689 CrIoSq.Cqid = Index;
690 CrIoSq.Qprio = 0;
691 CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
692 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
693
694 Status = Private->Passthru.PassThru (
695 &Private->Passthru,
696 0,
697 &CommandPacket,
698 NULL
699 );
700 if (EFI_ERROR (Status)) {
701 break;
702 }
703 }
704
705 Private->CreateIoQueue = FALSE;
706
707 return Status;
708 }
709
710 /**
711 Initialize the Nvm Express controller.
712
713 @param[in] Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
714
715 @retval EFI_SUCCESS The NVM Express Controller is initialized successfully.
716 @retval Others A device error occurred while initializing the controller.
717
718 **/
719 EFI_STATUS
720 NvmeControllerInit (
721 IN NVME_CONTROLLER_PRIVATE_DATA *Private
722 )
723 {
724 EFI_STATUS Status;
725 EFI_PCI_IO_PROTOCOL *PciIo;
726 UINT64 Supports;
727 NVME_AQA Aqa;
728 NVME_ASQ Asq;
729 NVME_ACQ Acq;
730 UINT8 Sn[21];
731 UINT8 Mn[41];
732 //
733 // Save original PCI attributes and enable this controller.
734 //
735 PciIo = Private->PciIo;
736 Status = PciIo->Attributes (
737 PciIo,
738 EfiPciIoAttributeOperationGet,
739 0,
740 &Private->PciAttributes
741 );
742
743 if (EFI_ERROR (Status)) {
744 return Status;
745 }
746
747 Status = PciIo->Attributes (
748 PciIo,
749 EfiPciIoAttributeOperationSupported,
750 0,
751 &Supports
752 );
753
754 if (!EFI_ERROR (Status)) {
755 Supports &= (UINT64)EFI_PCI_DEVICE_ENABLE;
756 Status = PciIo->Attributes (
757 PciIo,
758 EfiPciIoAttributeOperationEnable,
759 Supports,
760 NULL
761 );
762 }
763
764 if (EFI_ERROR (Status)) {
765 DEBUG ((EFI_D_INFO, "NvmeControllerInit: failed to enable controller\n"));
766 return Status;
767 }
768
769 //
770 // Enable 64-bit DMA support in the PCI layer.
771 //
772 Status = PciIo->Attributes (
773 PciIo,
774 EfiPciIoAttributeOperationEnable,
775 EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE,
776 NULL
777 );
778 if (EFI_ERROR (Status)) {
779 DEBUG ((EFI_D_WARN, "NvmeControllerInit: failed to enable 64-bit DMA (%r)\n", Status));
780 }
781
782 //
783 // Read the Controller Capabilities register and verify that the NVM command set is supported
784 //
785 Status = ReadNvmeControllerCapabilities (Private, &Private->Cap);
786 if (EFI_ERROR (Status)) {
787 return Status;
788 }
789
790 if (Private->Cap.Css != 0x01) {
791 DEBUG ((EFI_D_INFO, "NvmeControllerInit: the controller doesn't support NVMe command set\n"));
792 return EFI_UNSUPPORTED;
793 }
794
795 //
796 // Currently the driver only supports 4k page size.
797 //
798 ASSERT ((Private->Cap.Mpsmin + 12) <= EFI_PAGE_SHIFT);
799
800 Private->Cid[0] = 0;
801 Private->Cid[1] = 0;
802 Private->Cid[2] = 0;
803 Private->Pt[0] = 0;
804 Private->Pt[1] = 0;
805 Private->Pt[2] = 0;
806 Private->SqTdbl[0].Sqt = 0;
807 Private->SqTdbl[1].Sqt = 0;
808 Private->SqTdbl[2].Sqt = 0;
809 Private->CqHdbl[0].Cqh = 0;
810 Private->CqHdbl[1].Cqh = 0;
811 Private->CqHdbl[2].Cqh = 0;
812 Private->AsyncSqHead = 0;
813
814 Status = NvmeDisableController (Private);
815
816 if (EFI_ERROR(Status)) {
817 return Status;
818 }
819
820 //
821 // set number of entries admin submission & completion queues.
822 //
823 Aqa.Asqs = NVME_ASQ_SIZE;
824 Aqa.Rsvd1 = 0;
825 Aqa.Acqs = NVME_ACQ_SIZE;
826 Aqa.Rsvd2 = 0;
827
828 //
829 // Address of admin submission queue.
830 //
831 Asq = (UINT64)(UINTN)(Private->BufferPciAddr) & ~0xFFF;
832
833 //
834 // Address of admin completion queue.
835 //
836 Acq = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) & ~0xFFF;
837
838 //
839 // Address of I/O submission & completion queue.
840 //
841 ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (6));
842 Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer);
843 Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr);
844 Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE);
845 Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 1 * EFI_PAGE_SIZE);
846 Private->SqBuffer[1] = (NVME_SQ *)(UINTN)(Private->Buffer + 2 * EFI_PAGE_SIZE);
847 Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE);
848 Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE);
849 Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE);
850 Private->SqBuffer[2] = (NVME_SQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE);
851 Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE);
852 Private->CqBuffer[2] = (NVME_CQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE);
853 Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE);
854
855 DEBUG ((EFI_D_INFO, "Private->Buffer = [%016X]\n", (UINT64)(UINTN)Private->Buffer));
856 DEBUG ((EFI_D_INFO, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));
857 DEBUG ((EFI_D_INFO, "Admin Completion Queue size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));
858 DEBUG ((EFI_D_INFO, "Admin Submission Queue (SqBuffer[0]) = [%016X]\n", Private->SqBuffer[0]));
859 DEBUG ((EFI_D_INFO, "Admin Completion Queue (CqBuffer[0]) = [%016X]\n", Private->CqBuffer[0]));
860 DEBUG ((EFI_D_INFO, "Sync I/O Submission Queue (SqBuffer[1]) = [%016X]\n", Private->SqBuffer[1]));
861 DEBUG ((EFI_D_INFO, "Sync I/O Completion Queue (CqBuffer[1]) = [%016X]\n", Private->CqBuffer[1]));
862 DEBUG ((EFI_D_INFO, "Async I/O Submission Queue (SqBuffer[2]) = [%016X]\n", Private->SqBuffer[2]));
863 DEBUG ((EFI_D_INFO, "Async I/O Completion Queue (CqBuffer[2]) = [%016X]\n", Private->CqBuffer[2]));
864
865 //
866 // Program admin queue attributes.
867 //
868 Status = WriteNvmeAdminQueueAttributes (Private, &Aqa);
869
870 if (EFI_ERROR(Status)) {
871 return Status;
872 }
873
874 //
875 // Program admin submission queue address.
876 //
877 Status = WriteNvmeAdminSubmissionQueueBaseAddress (Private, &Asq);
878
879 if (EFI_ERROR(Status)) {
880 return Status;
881 }
882
883 //
884 // Program admin completion queue address.
885 //
886 Status = WriteNvmeAdminCompletionQueueBaseAddress (Private, &Acq);
887
888 if (EFI_ERROR(Status)) {
889 return Status;
890 }
891
892 Status = NvmeEnableController (Private);
893 if (EFI_ERROR(Status)) {
894 return Status;
895 }
896
897 //
898 // Allocate buffer for Identify Controller data
899 //
900 if (Private->ControllerData == NULL) {
901 Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof(NVME_ADMIN_CONTROLLER_DATA));
902
903 if (Private->ControllerData == NULL) {
904 return EFI_OUT_OF_RESOURCES;
905 }
906 }
907
908 //
909 // Get current Identify Controller Data
910 //
911 Status = NvmeIdentifyController (Private, Private->ControllerData);
912
913 if (EFI_ERROR(Status)) {
914 FreePool(Private->ControllerData);
915 Private->ControllerData = NULL;
916 return EFI_NOT_FOUND;
917 }
918
919 //
920 // Dump NvmExpress Identify Controller Data
921 //
922 CopyMem (Sn, Private->ControllerData->Sn, sizeof (Private->ControllerData->Sn));
923 Sn[20] = 0;
924 CopyMem (Mn, Private->ControllerData->Mn, sizeof (Private->ControllerData->Mn));
925 Mn[40] = 0;
926 DEBUG ((EFI_D_INFO, " == NVME IDENTIFY CONTROLLER DATA ==\n"));
927 DEBUG ((EFI_D_INFO, " PCI VID : 0x%x\n", Private->ControllerData->Vid));
928 DEBUG ((EFI_D_INFO, " PCI SSVID : 0x%x\n", Private->ControllerData->Ssvid));
929 DEBUG ((EFI_D_INFO, " SN : %a\n", Sn));
930 DEBUG ((EFI_D_INFO, " MN : %a\n", Mn));
931 DEBUG ((EFI_D_INFO, " FR : 0x%x\n", *((UINT64*)Private->ControllerData->Fr)));
932 DEBUG ((EFI_D_INFO, " RAB : 0x%x\n", Private->ControllerData->Rab));
933 DEBUG ((EFI_D_INFO, " IEEE : 0x%x\n", *(UINT32*)Private->ControllerData->Ieee_oui));
934 DEBUG ((EFI_D_INFO, " AERL : 0x%x\n", Private->ControllerData->Aerl));
935 DEBUG ((EFI_D_INFO, " SQES : 0x%x\n", Private->ControllerData->Sqes));
936 DEBUG ((EFI_D_INFO, " CQES : 0x%x\n", Private->ControllerData->Cqes));
937 DEBUG ((EFI_D_INFO, " NN : 0x%x\n", Private->ControllerData->Nn));
938
939 //
940 // Create two I/O completion queues.
941 // One for blocking I/O, one for non-blocking I/O.
942 //
943 Status = NvmeCreateIoCompletionQueue (Private);
944 if (EFI_ERROR(Status)) {
945 return Status;
946 }
947
948 //
949 // Create two I/O Submission queues.
950 // One for blocking I/O, one for non-blocking I/O.
951 //
952 Status = NvmeCreateIoSubmissionQueue (Private);
953
954 return Status;
955 }
956
957 /**
958 This routine is called to properly shutdown the Nvm Express controller per NVMe spec.
959
960 @param[in] ResetType The type of reset to perform.
961 @param[in] ResetStatus The status code for the reset.
962 @param[in] DataSize The size, in bytes, of ResetData.
963 @param[in] ResetData For a ResetType of EfiResetCold, EfiResetWarm, or
964 EfiResetShutdown the data buffer starts with a Null-terminated
965 string, optionally followed by additional binary data.
966 The string is a description that the caller may use to further
967 indicate the reason for the system reset. ResetData is only
968 valid if ResetStatus is something other than EFI_SUCCESS
969 unless the ResetType is EfiResetPlatformSpecific
970 where a minimum amount of ResetData is always required.
971 For a ResetType of EfiResetPlatformSpecific the data buffer
972 also starts with a Null-terminated string that is followed
973 by an EFI_GUID that describes the specific type of reset to perform.
974 **/
975 VOID
976 EFIAPI
977 NvmeShutdownAllControllers (
978 IN EFI_RESET_TYPE ResetType,
979 IN EFI_STATUS ResetStatus,
980 IN UINTN DataSize,
981 IN VOID *ResetData OPTIONAL
982 )
983 {
984 EFI_STATUS Status;
985 EFI_HANDLE *Handles;
986 UINTN HandleCount;
987 UINTN HandleIndex;
988 EFI_OPEN_PROTOCOL_INFORMATION_ENTRY *OpenInfos;
989 UINTN OpenInfoCount;
990 UINTN OpenInfoIndex;
991 EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *NvmePassThru;
992 NVME_CC Cc;
993 NVME_CSTS Csts;
994 UINTN Index;
995 NVME_CONTROLLER_PRIVATE_DATA *Private;
996
997 Status = gBS->LocateHandleBuffer (
998 ByProtocol,
999 &gEfiPciIoProtocolGuid,
1000 NULL,
1001 &HandleCount,
1002 &Handles
1003 );
1004 if (EFI_ERROR (Status)) {
1005 HandleCount = 0;
1006 }
1007
1008 for (HandleIndex = 0; HandleIndex < HandleCount; HandleIndex++) {
1009 Status = gBS->OpenProtocolInformation (
1010 Handles[HandleIndex],
1011 &gEfiPciIoProtocolGuid,
1012 &OpenInfos,
1013 &OpenInfoCount
1014 );
1015 if (EFI_ERROR (Status)) {
1016 continue;
1017 }
1018
1019 for (OpenInfoIndex = 0; OpenInfoIndex < OpenInfoCount; OpenInfoIndex++) {
1020 //
1021 // Find all the NVME controller managed by this driver.
1022 // gImageHandle equals to DriverBinding handle for this driver.
1023 //
1024 if (((OpenInfos[OpenInfoIndex].Attributes & EFI_OPEN_PROTOCOL_BY_DRIVER) != 0) &&
1025 (OpenInfos[OpenInfoIndex].AgentHandle == gImageHandle)) {
1026 Status = gBS->OpenProtocol (
1027 OpenInfos[OpenInfoIndex].ControllerHandle,
1028 &gEfiNvmExpressPassThruProtocolGuid,
1029 (VOID **) &NvmePassThru,
1030 NULL,
1031 NULL,
1032 EFI_OPEN_PROTOCOL_GET_PROTOCOL
1033 );
1034 if (EFI_ERROR (Status)) {
1035 continue;
1036 }
1037 Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (NvmePassThru);
1038
1039 //
1040 // Read Controller Configuration Register.
1041 //
1042 Status = ReadNvmeControllerConfiguration (Private, &Cc);
1043 if (EFI_ERROR(Status)) {
1044 continue;
1045 }
1046 //
1047 // The host should set the Shutdown Notification (CC.SHN) field to 01b
1048 // to indicate a normal shutdown operation.
1049 //
1050 Cc.Shn = NVME_CC_SHN_NORMAL_SHUTDOWN;
1051 Status = WriteNvmeControllerConfiguration (Private, &Cc);
1052 if (EFI_ERROR(Status)) {
1053 continue;
1054 }
1055
1056 //
1057 // The controller indicates when shutdown processing is completed by updating the
1058 // Shutdown Status (CSTS.SHST) field to 10b.
1059 // Wait up to 45 seconds (break down to 4500 x 10ms) for the shutdown to complete.
1060 //
1061 for (Index = 0; Index < NVME_SHUTDOWN_PROCESS_TIMEOUT * 100; Index++) {
1062 Status = ReadNvmeControllerStatus (Private, &Csts);
1063 if (!EFI_ERROR(Status) && (Csts.Shst == NVME_CSTS_SHST_SHUTDOWN_COMPLETED)) {
1064 DEBUG((DEBUG_INFO, "NvmeShutdownController: shutdown processing is completed after %dms.\n", Index * 10));
1065 break;
1066 }
1067 //
1068 // Stall for 10ms
1069 //
1070 gBS->Stall (10 * 1000);
1071 }
1072
1073 if (Index == NVME_SHUTDOWN_PROCESS_TIMEOUT * 100) {
1074 DEBUG((DEBUG_ERROR, "NvmeShutdownController: shutdown processing is timed out\n"));
1075 }
1076 }
1077 }
1078 }
1079 }
1080
1081 /**
1082 Register the shutdown notification through the ResetNotification protocol.
1083
1084 Register the shutdown notification when mNvmeControllerNumber increased from 0 to 1.
1085 **/
1086 VOID
1087 NvmeRegisterShutdownNotification (
1088 VOID
1089 )
1090 {
1091 EFI_STATUS Status;
1092 EFI_RESET_NOTIFICATION_PROTOCOL *ResetNotify;
1093
1094 mNvmeControllerNumber++;
1095 if (mNvmeControllerNumber == 1) {
1096 Status = gBS->LocateProtocol (&gEfiResetNotificationProtocolGuid, NULL, (VOID **) &ResetNotify);
1097 if (!EFI_ERROR (Status)) {
1098 Status = ResetNotify->RegisterResetNotify (ResetNotify, NvmeShutdownAllControllers);
1099 ASSERT_EFI_ERROR (Status);
1100 } else {
1101 DEBUG ((DEBUG_WARN, "NVME: ResetNotification absent! Shutdown notification cannot be performed!\n"));
1102 }
1103 }
1104 }
1105
1106 /**
1107 Unregister the shutdown notification through the ResetNotification protocol.
1108
1109 Unregister the shutdown notification when mNvmeControllerNumber decreased from 1 to 0.
1110 **/
1111 VOID
1112 NvmeUnregisterShutdownNotification (
1113 VOID
1114 )
1115 {
1116 EFI_STATUS Status;
1117 EFI_RESET_NOTIFICATION_PROTOCOL *ResetNotify;
1118
1119 mNvmeControllerNumber--;
1120 if (mNvmeControllerNumber == 0) {
1121 Status = gBS->LocateProtocol (&gEfiResetNotificationProtocolGuid, NULL, (VOID **) &ResetNotify);
1122 if (!EFI_ERROR (Status)) {
1123 Status = ResetNotify->UnregisterResetNotify (ResetNotify, NvmeShutdownAllControllers);
1124 ASSERT_EFI_ERROR (Status);
1125 }
1126 }
1127 }
EFI Development Kit II mirror for PVE