]> git.proxmox.com Git - mirror_edk2.git/blob - SecurityPkg/Tcg/Opal/OpalPassword/OpalNvmeMode.c
projects / mirror_edk2.git / blob
? search:


7657bb26e23c88df59bd3d4efe043e24946ffc60
[mirror_edk2.git] / SecurityPkg / Tcg / Opal / OpalPassword / OpalNvmeMode.c
1 /** @file
2 Provide functions to initialize NVME controller and perform NVME commands
3
4 Copyright (c) 2016 - 2018, Intel Corporation. All rights reserved.<BR>
5 This program and the accompanying materials
6 are licensed and made available under the terms and conditions of the BSD License
7 which accompanies this distribution. The full text of the license may be found at
8 http://opensource.org/licenses/bsd-license.php
9
10 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
12
13 **/
14
15 #include "OpalPasswordPei.h"
16
17
18 #define ALIGN(v, a) (UINTN)((((v) - 1) | ((a) - 1)) + 1)
19
20 ///
21 /// NVME Host controller registers operation
22 ///
23 #define NVME_GET_CAP(Nvme, Cap) NvmeMmioRead (Cap, Nvme->Nbar + NVME_CAP_OFFSET, sizeof (NVME_CAP))
24 #define NVME_GET_CC(Nvme, Cc) NvmeMmioRead (Cc, Nvme->Nbar + NVME_CC_OFFSET, sizeof (NVME_CC))
25 #define NVME_SET_CC(Nvme, Cc) NvmeMmioWrite (Nvme->Nbar + NVME_CC_OFFSET, Cc, sizeof (NVME_CC))
26 #define NVME_GET_CSTS(Nvme, Csts) NvmeMmioRead (Csts, Nvme->Nbar + NVME_CSTS_OFFSET, sizeof (NVME_CSTS))
27 #define NVME_GET_AQA(Nvme, Aqa) NvmeMmioRead (Aqa, Nvme->Nbar + NVME_AQA_OFFSET, sizeof (NVME_AQA))
28 #define NVME_SET_AQA(Nvme, Aqa) NvmeMmioWrite (Nvme->Nbar + NVME_AQA_OFFSET, Aqa, sizeof (NVME_AQA))
29 #define NVME_GET_ASQ(Nvme, Asq) NvmeMmioRead (Asq, Nvme->Nbar + NVME_ASQ_OFFSET, sizeof (NVME_ASQ))
30 #define NVME_SET_ASQ(Nvme, Asq) NvmeMmioWrite (Nvme->Nbar + NVME_ASQ_OFFSET, Asq, sizeof (NVME_ASQ))
31 #define NVME_GET_ACQ(Nvme, Acq) NvmeMmioRead (Acq, Nvme->Nbar + NVME_ACQ_OFFSET, sizeof (NVME_ACQ))
32 #define NVME_SET_ACQ(Nvme, Acq) NvmeMmioWrite (Nvme->Nbar + NVME_ACQ_OFFSET, Acq, sizeof (NVME_ACQ))
33 #define NVME_GET_VER(Nvme, Ver) NvmeMmioRead (Ver, Nvme->Nbar + NVME_VER_OFFSET, sizeof (NVME_VER))
34 #define NVME_SET_SQTDBL(Nvme, Qid, Sqtdbl) NvmeMmioWrite (Nvme->Nbar + NVME_SQTDBL_OFFSET(Qid, Nvme->Cap.Dstrd), Sqtdbl, sizeof (NVME_SQTDBL))
35 #define NVME_SET_CQHDBL(Nvme, Qid, Cqhdbl) NvmeMmioWrite (Nvme->Nbar + NVME_CQHDBL_OFFSET(Qid, Nvme->Cap.Dstrd), Cqhdbl, sizeof (NVME_CQHDBL))
36
37 ///
38 /// Base memory address
39 ///
40 enum {
41 BASEMEM_CONTROLLER_DATA,
42 BASEMEM_IDENTIFY_DATA,
43 BASEMEM_ASQ,
44 BASEMEM_ACQ,
45 BASEMEM_SQ,
46 BASEMEM_CQ,
47 BASEMEM_PRP,
48 BASEMEM_SECURITY,
49 MAX_BASEMEM_COUNT
50 };
51
52 ///
53 /// All of base memories are 4K(0x1000) alignment
54 ///
55 #define NVME_MEM_BASE(Nvme) ((UINTN)(Nvme->BaseMem))
56 #define NVME_CONTROL_DATA_BASE(Nvme) (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_CONTROLLER_DATA)) * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
57 #define NVME_NAMESPACE_DATA_BASE(Nvme) (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_IDENTIFY_DATA)) * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
58 #define NVME_ASQ_BASE(Nvme) (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_ASQ)) * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
59 #define NVME_ACQ_BASE(Nvme) (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_ACQ)) * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
60 #define NVME_SQ_BASE(Nvme, index) (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_SQ) + ((index)*(NVME_MAX_IO_QUEUES-1))) * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
61 #define NVME_CQ_BASE(Nvme, index) (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_CQ) + ((index)*(NVME_MAX_IO_QUEUES-1))) * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
62 #define NVME_PRP_BASE(Nvme, index) (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_PRP) + ((index)*NVME_PRP_SIZE)) * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
63 #define NVME_SEC_BASE(Nvme) (ALIGN (NVME_MEM_BASE(Nvme) + ((NvmeGetBaseMemPages (BASEMEM_SECURITY)) * EFI_PAGE_SIZE), EFI_PAGE_SIZE))
64
65 /**
66 Transfer MMIO Data to memory.
67
68 @param[in,out] MemBuffer - Destination: Memory address
69 @param[in] MmioAddr - Source: MMIO address
70 @param[in] Size - Size for read
71
72 @retval EFI_SUCCESS - MMIO read sucessfully
73 **/
74 EFI_STATUS
75 NvmeMmioRead (
76 IN OUT VOID *MemBuffer,
77 IN UINTN MmioAddr,
78 IN UINTN Size
79 )
80 {
81 UINTN Offset;
82 UINT8 Data;
83 UINT8 *Ptr;
84
85 // priority has adjusted
86 switch (Size) {
87 case 4:
88 *((UINT32 *)MemBuffer) = MmioRead32 (MmioAddr);
89 break;
90
91 case 8:
92 *((UINT64 *)MemBuffer) = MmioRead64 (MmioAddr);
93 break;
94
95 case 2:
96 *((UINT16 *)MemBuffer) = MmioRead16 (MmioAddr);
97 break;
98
99 case 1:
100 *((UINT8 *)MemBuffer) = MmioRead8 (MmioAddr);
101 break;
102
103 default:
104 Ptr = (UINT8 *)MemBuffer;
105 for (Offset = 0; Offset < Size; Offset += 1) {
106 Data = MmioRead8 (MmioAddr + Offset);
107 Ptr[Offset] = Data;
108 }
109 break;
110 }
111
112 return EFI_SUCCESS;
113 }
114
115 /**
116 Transfer memory data to MMIO.
117
118 @param[in,out] MmioAddr - Destination: MMIO address
119 @param[in] MemBuffer - Source: Memory address
120 @param[in] Size - Size for write
121
122 @retval EFI_SUCCESS - MMIO write sucessfully
123 **/
124 EFI_STATUS
125 NvmeMmioWrite (
126 IN OUT UINTN MmioAddr,
127 IN VOID *MemBuffer,
128 IN UINTN Size
129 )
130 {
131 UINTN Offset;
132 UINT8 Data;
133 UINT8 *Ptr;
134
135 // priority has adjusted
136 switch (Size) {
137 case 4:
138 MmioWrite32 (MmioAddr, *((UINT32 *)MemBuffer));
139 break;
140
141 case 8:
142 MmioWrite64 (MmioAddr, *((UINT64 *)MemBuffer));
143 break;
144
145 case 2:
146 MmioWrite16 (MmioAddr, *((UINT16 *)MemBuffer));
147 break;
148
149 case 1:
150 MmioWrite8 (MmioAddr, *((UINT8 *)MemBuffer));
151 break;
152
153 default:
154 Ptr = (UINT8 *)MemBuffer;
155 for (Offset = 0; Offset < Size; Offset += 1) {
156 Data = Ptr[Offset];
157 MmioWrite8 (MmioAddr + Offset, Data);
158 }
159 break;
160 }
161
162 return EFI_SUCCESS;
163 }
164
165 /**
166 Transfer MMIO data to memory.
167
168 @param[in,out] MemBuffer - Destination: Memory address
169 @param[in] MmioAddr - Source: MMIO address
170 @param[in] Size - Size for read
171
172 @retval EFI_SUCCESS - MMIO read sucessfully
173 **/
174 EFI_STATUS
175 OpalPciRead (
176 IN OUT VOID *MemBuffer,
177 IN UINTN MmioAddr,
178 IN UINTN Size
179 )
180 {
181 UINTN Offset;
182 UINT8 Data;
183 UINT8 *Ptr;
184
185 // priority has adjusted
186 switch (Size) {
187 case 4:
188 *((UINT32 *)MemBuffer) = PciRead32 (MmioAddr);
189 break;
190
191 case 2:
192 *((UINT16 *)MemBuffer) = PciRead16 (MmioAddr);
193 break;
194
195 case 1:
196 *((UINT8 *)MemBuffer) = PciRead8 (MmioAddr);
197 break;
198
199 default:
200 Ptr = (UINT8 *)MemBuffer;
201 for (Offset = 0; Offset < Size; Offset += 1) {
202 Data = PciRead8 (MmioAddr + Offset);
203 Ptr[Offset] = Data;
204 }
205 break;
206 }
207
208 return EFI_SUCCESS;
209 }
210
211 /**
212 Transfer memory data to MMIO.
213
214 @param[in,out] MmioAddr - Destination: MMIO address
215 @param[in] MemBuffer - Source: Memory address
216 @param[in] Size - Size for write
217
218 @retval EFI_SUCCESS - MMIO write sucessfully
219 **/
220 EFI_STATUS
221 OpalPciWrite (
222 IN OUT UINTN MmioAddr,
223 IN VOID *MemBuffer,
224 IN UINTN Size
225 )
226 {
227 UINTN Offset;
228 UINT8 Data;
229 UINT8 *Ptr;
230
231 // priority has adjusted
232 switch (Size) {
233 case 4:
234 PciWrite32 (MmioAddr, *((UINT32 *)MemBuffer));
235 break;
236
237 case 2:
238 PciWrite16 (MmioAddr, *((UINT16 *)MemBuffer));
239 break;
240
241 case 1:
242 PciWrite8 (MmioAddr, *((UINT8 *)MemBuffer));
243 break;
244
245 default:
246 Ptr = (UINT8 *)MemBuffer;
247 for (Offset = 0; Offset < Size; Offset += 1) {
248 Data = Ptr[Offset];
249 PciWrite8 (MmioAddr + Offset, Data);
250 }
251 break;
252 }
253
254 return EFI_SUCCESS;
255 }
256
257 /**
258 Get total pages for specific NVME based memory.
259
260 @param[in] BaseMemIndex - The Index of BaseMem (0-based).
261
262 @retval - The page count for specific BaseMem Index
263
264 **/
265 UINT32
266 NvmeGetBaseMemPages (
267 IN UINTN BaseMemIndex
268 )
269 {
270 UINT32 Pages;
271 UINTN Index;
272 UINT32 PageSizeList[8];
273
274 PageSizeList[0] = 1; /* Controller Data */
275 PageSizeList[1] = 1; /* Identify Data */
276 PageSizeList[2] = 1; /* ASQ */
277 PageSizeList[3] = 1; /* ACQ */
278 PageSizeList[4] = 1; /* SQs */
279 PageSizeList[5] = 1; /* CQs */
280 PageSizeList[6] = NVME_PRP_SIZE * NVME_CSQ_DEPTH; /* PRPs */
281 PageSizeList[7] = 1; /* Security Commands */
282
283 if (BaseMemIndex > MAX_BASEMEM_COUNT) {
284 ASSERT (FALSE);
285 return 0;
286 }
287
288 Pages = 0;
289 for (Index = 0; Index < BaseMemIndex; Index++) {
290 Pages += PageSizeList[Index];
291 }
292
293 return Pages;
294 }
295
296 /**
297 Wait for NVME controller status to be ready or not.
298
299 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
300 @param[in] WaitReady - Flag for waitting status ready or not
301
302 @return EFI_SUCCESS - Successfully to wait specific status.
303 @return others - Fail to wait for specific controller status.
304
305 **/
306 STATIC
307 EFI_STATUS
308 NvmeWaitController (
309 IN NVME_CONTEXT *Nvme,
310 IN BOOLEAN WaitReady
311 )
312 {
313 NVME_CSTS Csts;
314 EFI_STATUS Status;
315 UINT32 Index;
316 UINT8 Timeout;
317
318 //
319 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
320 // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
321 //
322 if (Nvme->Cap.To == 0) {
323 Timeout = 1;
324 } else {
325 Timeout = Nvme->Cap.To;
326 }
327
328 Status = EFI_SUCCESS;
329 for(Index = (Timeout * 500); Index != 0; --Index) {
330 MicroSecondDelay (1000);
331
332 //
333 // Check if the controller is initialized
334 //
335 Status = NVME_GET_CSTS (Nvme, &Csts);
336 if (EFI_ERROR(Status)) {
337 DEBUG ((DEBUG_ERROR, "NVME_GET_CSTS fail, Status = %r\n", Status));
338 return Status;
339 }
340
341 if ((BOOLEAN) Csts.Rdy == WaitReady) {
342 break;
343 }
344 }
345
346 if (Index == 0) {
347 Status = EFI_TIMEOUT;
348 }
349
350 return Status;
351 }
352
353 /**
354 Disable the Nvm Express controller.
355
356 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
357
358 @return EFI_SUCCESS - Successfully disable the controller.
359 @return others - Fail to disable the controller.
360
361 **/
362 STATIC
363 EFI_STATUS
364 NvmeDisableController (
365 IN NVME_CONTEXT *Nvme
366 )
367 {
368 NVME_CC Cc;
369 NVME_CSTS Csts;
370 EFI_STATUS Status;
371
372 Status = NVME_GET_CSTS (Nvme, &Csts);
373
374 ///
375 /// Read Controller Configuration Register.
376 ///
377 Status = NVME_GET_CC (Nvme, &Cc);
378 if (EFI_ERROR(Status)) {
379 DEBUG ((DEBUG_ERROR, "NVME_GET_CC fail, Status = %r\n", Status));
380 goto Done;
381 }
382
383 if (Cc.En == 1) {
384 Cc.En = 0;
385 ///
386 /// Disable the controller.
387 ///
388 Status = NVME_SET_CC (Nvme, &Cc);
389 if (EFI_ERROR(Status)) {
390 DEBUG ((DEBUG_ERROR, "NVME_SET_CC fail, Status = %r\n", Status));
391 goto Done;
392 }
393 }
394
395 Status = NvmeWaitController (Nvme, FALSE);
396 if (EFI_ERROR(Status)) {
397 DEBUG ((DEBUG_ERROR, "NvmeWaitController fail, Status = %r\n", Status));
398 goto Done;
399 }
400
401 return EFI_SUCCESS;
402
403 Done:
404 DEBUG ((DEBUG_INFO, "NvmeDisableController fail, Status: %r\n", Status));
405 return Status;
406 }
407
408 /**
409 Enable the Nvm Express controller.
410
411 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
412
413 @return EFI_SUCCESS - Successfully enable the controller.
414 @return EFI_DEVICE_ERROR - Fail to enable the controller.
415 @return EFI_TIMEOUT - Fail to enable the controller in given time slot.
416
417 **/
418 STATIC
419 EFI_STATUS
420 NvmeEnableController (
421 IN NVME_CONTEXT *Nvme
422 )
423 {
424 NVME_CC Cc;
425 EFI_STATUS Status;
426
427 //
428 // Enable the controller
429 //
430 ZeroMem (&Cc, sizeof (NVME_CC));
431 Cc.En = 1;
432 Cc.Iosqes = 6;
433 Cc.Iocqes = 4;
434 Status = NVME_SET_CC (Nvme, &Cc);
435 if (EFI_ERROR(Status)) {
436 DEBUG ((DEBUG_ERROR, "NVME_SET_CC fail, Status = %r\n", Status));
437 goto Done;
438 }
439
440 Status = NvmeWaitController (Nvme, TRUE);
441 if (EFI_ERROR(Status)) {
442 DEBUG ((DEBUG_ERROR, "NvmeWaitController fail, Status = %r\n", Status));
443 goto Done;
444 }
445
446 return EFI_SUCCESS;
447
448 Done:
449 DEBUG ((DEBUG_INFO, "NvmeEnableController fail, Status: %r\n", Status));
450 return Status;
451 }
452
453 /**
454 Shutdown the Nvm Express controller.
455
456 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
457
458 @return EFI_SUCCESS - Successfully shutdown the controller.
459 @return EFI_DEVICE_ERROR - Fail to shutdown the controller.
460 @return EFI_TIMEOUT - Fail to shutdown the controller in given time slot.
461
462 **/
463 STATIC
464 EFI_STATUS
465 NvmeShutdownController (
466 IN NVME_CONTEXT *Nvme
467 )
468 {
469 NVME_CC Cc;
470 NVME_CSTS Csts;
471 EFI_STATUS Status;
472 UINT32 Index;
473 UINTN Timeout;
474
475 Status = NVME_GET_CC (Nvme, &Cc);
476 if (EFI_ERROR(Status)) {
477 DEBUG ((DEBUG_ERROR, "NVME_GET_CC fail, Status = %r\n", Status));
478 return Status;
479 }
480
481 Cc.Shn = 1; // Normal shutdown
482
483 Status = NVME_SET_CC (Nvme, &Cc);
484 if (EFI_ERROR(Status)) {
485 DEBUG ((DEBUG_ERROR, "NVME_SET_CC fail, Status = %r\n", Status));
486 return Status;
487 }
488
489 Timeout = NVME_GENERIC_TIMEOUT/1000; // ms
490 for(Index = (UINT32)(Timeout); Index != 0; --Index) {
491 MicroSecondDelay (1000);
492
493 Status = NVME_GET_CSTS (Nvme, &Csts);
494 if (EFI_ERROR(Status)) {
495 DEBUG ((DEBUG_ERROR, "NVME_GET_CSTS fail, Status = %r\n", Status));
496 return Status;
497 }
498
499 if (Csts.Shst == 2) { // Shutdown processing complete
500 break;
501 }
502 }
503
504 if (Index == 0) {
505 Status = EFI_TIMEOUT;
506 }
507
508 return Status;
509 }
510
511 /**
512 Check the execution status from a given completion queue entry.
513
514 @param[in] Cq - A pointer to the NVME_CQ item.
515
516 **/
517 EFI_STATUS
518 NvmeCheckCqStatus (
519 IN NVME_CQ *Cq
520 )
521 {
522 if (Cq->Sct == 0x0 && Cq->Sc == 0x0) {
523 return EFI_SUCCESS;
524 }
525
526 DEBUG ((DEBUG_INFO, "Dump NVMe Completion Entry Status from [0x%x]:\n", (UINTN)Cq));
527 DEBUG ((DEBUG_INFO, " SQ Identifier : [0x%x], Phase Tag : [%d], Cmd Identifier : [0x%x]\n", Cq->Sqid, Cq->Pt, Cq->Cid));
528 DEBUG ((DEBUG_INFO, " NVMe Cmd Execution Result - "));
529
530 switch (Cq->Sct) {
531 case 0x0:
532 switch (Cq->Sc) {
533 case 0x0:
534 DEBUG ((DEBUG_INFO, "Successful Completion\n"));
535 return EFI_SUCCESS;
536 case 0x1:
537 DEBUG ((DEBUG_INFO, "Invalid Command Opcode\n"));
538 break;
539 case 0x2:
540 DEBUG ((DEBUG_INFO, "Invalid Field in Command\n"));
541 break;
542 case 0x3:
543 DEBUG ((DEBUG_INFO, "Command ID Conflict\n"));
544 break;
545 case 0x4:
546 DEBUG ((DEBUG_INFO, "Data Transfer Error\n"));
547 break;
548 case 0x5:
549 DEBUG ((DEBUG_INFO, "Commands Aborted due to Power Loss Notification\n"));
550 break;
551 case 0x6:
552 DEBUG ((DEBUG_INFO, "Internal Device Error\n"));
553 break;
554 case 0x7:
555 DEBUG ((DEBUG_INFO, "Command Abort Requested\n"));
556 break;
557 case 0x8:
558 DEBUG ((DEBUG_INFO, "Command Aborted due to SQ Deletion\n"));
559 break;
560 case 0x9:
561 DEBUG ((DEBUG_INFO, "Command Aborted due to Failed Fused Command\n"));
562 break;
563 case 0xA:
564 DEBUG ((DEBUG_INFO, "Command Aborted due to Missing Fused Command\n"));
565 break;
566 case 0xB:
567 DEBUG ((DEBUG_INFO, "Invalid Namespace or Format\n"));
568 break;
569 case 0xC:
570 DEBUG ((DEBUG_INFO, "Command Sequence Error\n"));
571 break;
572 case 0xD:
573 DEBUG ((DEBUG_INFO, "Invalid SGL Last Segment Descriptor\n"));
574 break;
575 case 0xE:
576 DEBUG ((DEBUG_INFO, "Invalid Number of SGL Descriptors\n"));
577 break;
578 case 0xF:
579 DEBUG ((DEBUG_INFO, "Data SGL Length Invalid\n"));
580 break;
581 case 0x10:
582 DEBUG ((DEBUG_INFO, "Metadata SGL Length Invalid\n"));
583 break;
584 case 0x11:
585 DEBUG ((DEBUG_INFO, "SGL Descriptor Type Invalid\n"));
586 break;
587 case 0x80:
588 DEBUG ((DEBUG_INFO, "LBA Out of Range\n"));
589 break;
590 case 0x81:
591 DEBUG ((DEBUG_INFO, "Capacity Exceeded\n"));
592 break;
593 case 0x82:
594 DEBUG ((DEBUG_INFO, "Namespace Not Ready\n"));
595 break;
596 case 0x83:
597 DEBUG ((DEBUG_INFO, "Reservation Conflict\n"));
598 break;
599 }
600 break;
601
602 case 0x1:
603 switch (Cq->Sc) {
604 case 0x0:
605 DEBUG ((DEBUG_INFO, "Completion Queue Invalid\n"));
606 break;
607 case 0x1:
608 DEBUG ((DEBUG_INFO, "Invalid Queue Identifier\n"));
609 break;
610 case 0x2:
611 DEBUG ((DEBUG_INFO, "Maximum Queue Size Exceeded\n"));
612 break;
613 case 0x3:
614 DEBUG ((DEBUG_INFO, "Abort Command Limit Exceeded\n"));
615 break;
616 case 0x5:
617 DEBUG ((DEBUG_INFO, "Asynchronous Event Request Limit Exceeded\n"));
618 break;
619 case 0x6:
620 DEBUG ((DEBUG_INFO, "Invalid Firmware Slot\n"));
621 break;
622 case 0x7:
623 DEBUG ((DEBUG_INFO, "Invalid Firmware Image\n"));
624 break;
625 case 0x8:
626 DEBUG ((DEBUG_INFO, "Invalid Interrupt Vector\n"));
627 break;
628 case 0x9:
629 DEBUG ((DEBUG_INFO, "Invalid Log Page\n"));
630 break;
631 case 0xA:
632 DEBUG ((DEBUG_INFO, "Invalid Format\n"));
633 break;
634 case 0xB:
635 DEBUG ((DEBUG_INFO, "Firmware Application Requires Conventional Reset\n"));
636 break;
637 case 0xC:
638 DEBUG ((DEBUG_INFO, "Invalid Queue Deletion\n"));
639 break;
640 case 0xD:
641 DEBUG ((DEBUG_INFO, "Feature Identifier Not Saveable\n"));
642 break;
643 case 0xE:
644 DEBUG ((DEBUG_INFO, "Feature Not Changeable\n"));
645 break;
646 case 0xF:
647 DEBUG ((DEBUG_INFO, "Feature Not Namespace Specific\n"));
648 break;
649 case 0x10:
650 DEBUG ((DEBUG_INFO, "Firmware Application Requires NVM Subsystem Reset\n"));
651 break;
652 case 0x80:
653 DEBUG ((DEBUG_INFO, "Conflicting Attributes\n"));
654 break;
655 case 0x81:
656 DEBUG ((DEBUG_INFO, "Invalid Protection Information\n"));
657 break;
658 case 0x82:
659 DEBUG ((DEBUG_INFO, "Attempted Write to Read Only Range\n"));
660 break;
661 }
662 break;
663
664 case 0x2:
665 switch (Cq->Sc) {
666 case 0x80:
667 DEBUG ((DEBUG_INFO, "Write Fault\n"));
668 break;
669 case 0x81:
670 DEBUG ((DEBUG_INFO, "Unrecovered Read Error\n"));
671 break;
672 case 0x82:
673 DEBUG ((DEBUG_INFO, "End-to-end Guard Check Error\n"));
674 break;
675 case 0x83:
676 DEBUG ((DEBUG_INFO, "End-to-end Application Tag Check Error\n"));
677 break;
678 case 0x84:
679 DEBUG ((DEBUG_INFO, "End-to-end Reference Tag Check Error\n"));
680 break;
681 case 0x85:
682 DEBUG ((DEBUG_INFO, "Compare Failure\n"));
683 break;
684 case 0x86:
685 DEBUG ((DEBUG_INFO, "Access Denied\n"));
686 break;
687 }
688 break;
689
690 default:
691 DEBUG ((DEBUG_INFO, "Unknown error\n"));
692 break;
693 }
694
695 return EFI_DEVICE_ERROR;
696 }
697
698 /**
699 Create PRP lists for Data transfer which is larger than 2 memory pages.
700 Note here we calcuate the number of required PRP lists and allocate them at one time.
701
702 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
703 @param[in] SqId - The SQ index for this PRP
704 @param[in] PhysicalAddr - The physical base address of Data Buffer.
705 @param[in] Pages - The number of pages to be transfered.
706 @param[out] PrpListHost - The host base address of PRP lists.
707 @param[in,out] PrpListNo - The number of PRP List.
708
709 @retval The pointer Value to the first PRP List of the PRP lists.
710
711 **/
712 STATIC
713 UINT64
714 NvmeCreatePrpList (
715 IN NVME_CONTEXT *Nvme,
716 IN UINT16 SqId,
717 IN EFI_PHYSICAL_ADDRESS PhysicalAddr,
718 IN UINTN Pages,
719 OUT VOID **PrpListHost,
720 IN OUT UINTN *PrpListNo
721 )
722 {
723 UINTN PrpEntryNo;
724 UINT64 PrpListBase;
725 UINTN PrpListIndex;
726 UINTN PrpEntryIndex;
727 UINT64 Remainder;
728 EFI_PHYSICAL_ADDRESS PrpListPhyAddr;
729 UINTN Bytes;
730 UINT8 *PrpEntry;
731 EFI_PHYSICAL_ADDRESS NewPhyAddr;
732
733 ///
734 /// The number of Prp Entry in a memory page.
735 ///
736 PrpEntryNo = EFI_PAGE_SIZE / sizeof (UINT64);
737
738 ///
739 /// Calculate total PrpList number.
740 ///
741 *PrpListNo = (UINTN) DivU64x64Remainder ((UINT64)Pages, (UINT64)PrpEntryNo, &Remainder);
742 if (Remainder != 0) {
743 *PrpListNo += 1;
744 }
745
746 if (*PrpListNo > NVME_PRP_SIZE) {
747 DEBUG ((DEBUG_INFO, "NvmeCreatePrpList (PhysicalAddr: %lx, Pages: %x) PrpEntryNo: %x\n",
748 PhysicalAddr, Pages, PrpEntryNo));
749 DEBUG ((DEBUG_INFO, "*PrpListNo: %x, Remainder: %lx", *PrpListNo, Remainder));
750 ASSERT (FALSE);
751 }
752 *PrpListHost = (VOID *)(UINTN) NVME_PRP_BASE (Nvme, SqId);
753
754 Bytes = EFI_PAGES_TO_SIZE (*PrpListNo);
755 PrpListPhyAddr = (UINT64)(UINTN)(*PrpListHost);
756
757 ///
758 /// Fill all PRP lists except of last one.
759 ///
760 ZeroMem (*PrpListHost, Bytes);
761 for (PrpListIndex = 0; PrpListIndex < *PrpListNo - 1; ++PrpListIndex) {
762 PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
763
764 for (PrpEntryIndex = 0; PrpEntryIndex < PrpEntryNo; ++PrpEntryIndex) {
765 PrpEntry = (UINT8 *)(UINTN) (PrpListBase + PrpEntryIndex * sizeof(UINT64));
766 if (PrpEntryIndex != PrpEntryNo - 1) {
767 ///
768 /// Fill all PRP entries except of last one.
769 ///
770 CopyMem (PrpEntry, (VOID *)(UINTN) (&PhysicalAddr), sizeof (UINT64));
771 PhysicalAddr += EFI_PAGE_SIZE;
772 } else {
773 ///
774 /// Fill last PRP entries with next PRP List pointer.
775 ///
776 NewPhyAddr = (PrpListPhyAddr + (PrpListIndex + 1) * EFI_PAGE_SIZE);
777 CopyMem (PrpEntry, (VOID *)(UINTN) (&NewPhyAddr), sizeof (UINT64));
778 }
779 }
780 }
781
782 ///
783 /// Fill last PRP list.
784 ///
785 PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
786 for (PrpEntryIndex = 0; PrpEntryIndex < ((Remainder != 0) ? Remainder : PrpEntryNo); ++PrpEntryIndex) {
787 PrpEntry = (UINT8 *)(UINTN) (PrpListBase + PrpEntryIndex * sizeof(UINT64));
788 CopyMem (PrpEntry, (VOID *)(UINTN) (&PhysicalAddr), sizeof (UINT64));
789
790 PhysicalAddr += EFI_PAGE_SIZE;
791 }
792
793 return PrpListPhyAddr;
794 }
795
796 /**
797 Check whether there are available command slots.
798
799 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
800 @param[in] Qid - Queue index
801
802 @retval EFI_SUCCESS - Available command slot is found
803 @retval EFI_NOT_READY - No available command slot is found
804 @retval EFI_DEVICE_ERROR - Error occurred on device side.
805
806 **/
807 EFI_STATUS
808 NvmeHasFreeCmdSlot (
809 IN NVME_CONTEXT *Nvme,
810 IN UINT8 Qid
811 )
812 {
813 return TRUE;
814 }
815
816 /**
817 Check whether all command slots are clean.
818
819 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
820 @param[in] Qid - Queue index
821
822 @retval EFI_SUCCESS - All command slots are clean
823 @retval EFI_NOT_READY - Not all command slots are clean
824 @retval EFI_DEVICE_ERROR - Error occurred on device side.
825
826 **/
827 EFI_STATUS
828 NvmeIsAllCmdSlotClean (
829 IN NVME_CONTEXT *Nvme,
830 IN UINT8 Qid
831 )
832 {
833 return EFI_SUCCESS;
834 }
835
836 /**
837 Waits until all NVME commands completed.
838
839 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
840 @param[in] Qid - Queue index
841
842 @retval EFI_SUCCESS - All NVME commands have completed
843 @retval EFI_TIMEOUT - Timeout occured
844 @retval EFI_NOT_READY - Not all NVME commands have completed
845 @retval others - Error occurred on device side.
846 **/
847 EFI_STATUS
848 NvmeWaitAllComplete (
849 IN NVME_CONTEXT *Nvme,
850 IN UINT8 Qid
851 )
852 {
853 return EFI_SUCCESS;
854 }
855
856 /**
857 Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function supports
858 both blocking I/O and nonblocking I/O. The blocking I/O functionality is required, and the nonblocking
859 I/O functionality is optional.
860
861 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
862 @param[in] NamespaceId - Is a 32 bit Namespace ID to which the Express HCI command packet will be sent.
863 A Value of 0 denotes the NVM Express controller, a Value of all 0FFh in the namespace
864 ID specifies that the command packet should be sent to all valid namespaces.
865 @param[in] NamespaceUuid - Is a 64 bit Namespace UUID to which the Express HCI command packet will be sent.
866 A Value of 0 denotes the NVM Express controller, a Value of all 0FFh in the namespace
867 UUID specifies that the command packet should be sent to all valid namespaces.
868 @param[in,out] Packet - A pointer to the NVM Express HCI Command Packet to send to the NVMe namespace specified
869 by NamespaceId.
870
871 @retval EFI_SUCCESS - The NVM Express Command Packet was sent by the host. TransferLength bytes were transferred
872 to, or from DataBuffer.
873 @retval EFI_NOT_READY - The NVM Express Command Packet could not be sent because the controller is not ready. The caller
874 may retry again later.
875 @retval EFI_DEVICE_ERROR - A device error occurred while attempting to send the NVM Express Command Packet.
876 @retval EFI_INVALID_PARAMETER - Namespace, or the contents of NVM_EXPRESS_PASS_THRU_COMMAND_PACKET are invalid. The NVM
877 Express Command Packet was not sent, so no additional status information is available.
878 @retval EFI_UNSUPPORTED - The command described by the NVM Express Command Packet is not supported by the host adapter.
879 The NVM Express Command Packet was not sent, so no additional status information is available.
880 @retval EFI_TIMEOUT - A timeout occurred while waiting for the NVM Express Command Packet to execute.
881
882 **/
883 EFI_STATUS
884 NvmePassThru (
885 IN NVME_CONTEXT *Nvme,
886 IN UINT32 NamespaceId,
887 IN UINT64 NamespaceUuid,
888 IN OUT NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *Packet
889 )
890 {
891 EFI_STATUS Status;
892 NVME_SQ *Sq;
893 NVME_CQ *Cq;
894 UINT8 Qid;
895 UINT32 Bytes;
896 UINT32 Offset;
897 EFI_PHYSICAL_ADDRESS PhyAddr;
898 VOID *PrpListHost;
899 UINTN PrpListNo;
900 UINT32 Timer;
901 UINTN SqSize;
902 UINTN CqSize;
903
904 ///
905 /// check the Data fields in Packet parameter.
906 ///
907 if ((Nvme == NULL) || (Packet == NULL)) {
908 DEBUG ((DEBUG_ERROR, "NvmePassThru, invalid parameter: Nvme(%x)/Packet(%x)\n",
909 (UINTN)Nvme, (UINTN)Packet));
910 return EFI_INVALID_PARAMETER;
911 }
912
913 if ((Packet->NvmeCmd == NULL) || (Packet->NvmeResponse == NULL)) {
914 DEBUG ((DEBUG_ERROR, "NvmePassThru, invalid parameter: NvmeCmd(%x)/NvmeResponse(%x)\n",
915 (UINTN)Packet->NvmeCmd, (UINTN)Packet->NvmeResponse));
916 return EFI_INVALID_PARAMETER;
917 }
918
919 if (Packet->QueueId != NVME_ADMIN_QUEUE && Packet->QueueId != NVME_IO_QUEUE) {
920 DEBUG ((DEBUG_ERROR, "NvmePassThru, invalid parameter: QueueId(%x)\n",
921 Packet->QueueId));
922 return EFI_INVALID_PARAMETER;
923 }
924
925 PrpListHost = NULL;
926 PrpListNo = 0;
927 Status = EFI_SUCCESS;
928
929 Qid = Packet->QueueId;
930 Sq = Nvme->SqBuffer[Qid] + Nvme->SqTdbl[Qid].Sqt;
931 Cq = Nvme->CqBuffer[Qid] + Nvme->CqHdbl[Qid].Cqh;
932 if (Qid == NVME_ADMIN_QUEUE) {
933 SqSize = NVME_ASQ_SIZE + 1;
934 CqSize = NVME_ACQ_SIZE + 1;
935 } else {
936 SqSize = NVME_CSQ_DEPTH;
937 CqSize = NVME_CCQ_DEPTH;
938 }
939
940 if (Packet->NvmeCmd->Nsid != NamespaceId) {
941 DEBUG ((DEBUG_ERROR, "NvmePassThru: Nsid mismatch (%x, %x)\n",
942 Packet->NvmeCmd->Nsid, NamespaceId));
943 return EFI_INVALID_PARAMETER;
944 }
945
946 ZeroMem (Sq, sizeof (NVME_SQ));
947 Sq->Opc = Packet->NvmeCmd->Cdw0.Opcode;
948 Sq->Fuse = Packet->NvmeCmd->Cdw0.FusedOperation;
949 Sq->Cid = Packet->NvmeCmd->Cdw0.Cid;
950 Sq->Nsid = Packet->NvmeCmd->Nsid;
951
952 ///
953 /// Currently we only support PRP for Data transfer, SGL is NOT supported.
954 ///
955 ASSERT (Sq->Psdt == 0);
956 if (Sq->Psdt != 0) {
957 DEBUG ((DEBUG_ERROR, "NvmePassThru: doesn't support SGL mechanism\n"));
958 return EFI_UNSUPPORTED;
959 }
960
961 Sq->Prp[0] = Packet->TransferBuffer;
962 Sq->Prp[1] = 0;
963
964 if(Packet->MetadataBuffer != (UINT64)(UINTN)NULL) {
965 Sq->Mptr = Packet->MetadataBuffer;
966 }
967
968 ///
969 /// If the Buffer Size spans more than two memory pages (page Size as defined in CC.Mps),
970 /// then build a PRP list in the second PRP submission queue entry.
971 ///
972 Offset = ((UINT32)Sq->Prp[0]) & (EFI_PAGE_SIZE - 1);
973 Bytes = Packet->TransferLength;
974
975 if ((Offset + Bytes) > (EFI_PAGE_SIZE * 2)) {
976 ///
977 /// Create PrpList for remaining Data Buffer.
978 ///
979 PhyAddr = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
980 Sq->Prp[1] = NvmeCreatePrpList (Nvme, Nvme->SqTdbl[Qid].Sqt, PhyAddr, EFI_SIZE_TO_PAGES(Offset + Bytes) - 1, &PrpListHost, &PrpListNo);
981 if (Sq->Prp[1] == 0) {
982 Status = EFI_OUT_OF_RESOURCES;
983 DEBUG ((DEBUG_ERROR, "NvmeCreatePrpList fail, Status: %r\n", Status));
984 goto EXIT;
985 }
986
987 } else if ((Offset + Bytes) > EFI_PAGE_SIZE) {
988 Sq->Prp[1] = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
989 }
990
991 if(Packet->NvmeCmd->Flags & CDW10_VALID) {
992 Sq->Payload.Raw.Cdw10 = Packet->NvmeCmd->Cdw10;
993 }
994 if(Packet->NvmeCmd->Flags & CDW11_VALID) {
995 Sq->Payload.Raw.Cdw11 = Packet->NvmeCmd->Cdw11;
996 }
997 if(Packet->NvmeCmd->Flags & CDW12_VALID) {
998 Sq->Payload.Raw.Cdw12 = Packet->NvmeCmd->Cdw12;
999 }
1000 if(Packet->NvmeCmd->Flags & CDW13_VALID) {
1001 Sq->Payload.Raw.Cdw13 = Packet->NvmeCmd->Cdw13;
1002 }
1003 if(Packet->NvmeCmd->Flags & CDW14_VALID) {
1004 Sq->Payload.Raw.Cdw14 = Packet->NvmeCmd->Cdw14;
1005 }
1006 if(Packet->NvmeCmd->Flags & CDW15_VALID) {
1007 Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
1008 }
1009
1010 ///
1011 /// Ring the submission queue doorbell.
1012 ///
1013 Nvme->SqTdbl[Qid].Sqt++;
1014 if(Nvme->SqTdbl[Qid].Sqt == SqSize) {
1015 Nvme->SqTdbl[Qid].Sqt = 0;
1016 }
1017 Status = NVME_SET_SQTDBL (Nvme, Qid, &Nvme->SqTdbl[Qid]);
1018 if (EFI_ERROR(Status)) {
1019 DEBUG ((DEBUG_ERROR, "NVME_SET_SQTDBL fail, Status: %r\n", Status));
1020 goto EXIT;
1021 }
1022
1023 ///
1024 /// Wait for completion queue to get filled in.
1025 ///
1026 Status = EFI_TIMEOUT;
1027 Timer = 0;
1028 while (Timer < NVME_CMD_TIMEOUT) {
1029 //DEBUG ((DEBUG_VERBOSE, "Timer: %x, Cq:\n", Timer));
1030 //DumpMem (Cq, sizeof (NVME_CQ));
1031 if (Cq->Pt != Nvme->Pt[Qid]) {
1032 Status = EFI_SUCCESS;
1033 break;
1034 }
1035
1036 MicroSecondDelay (NVME_CMD_WAIT);
1037 Timer += NVME_CMD_WAIT;
1038 }
1039
1040 Nvme->CqHdbl[Qid].Cqh++;
1041 if (Nvme->CqHdbl[Qid].Cqh == CqSize) {
1042 Nvme->CqHdbl[Qid].Cqh = 0;
1043 Nvme->Pt[Qid] ^= 1;
1044 }
1045
1046 ///
1047 /// Copy the Respose Queue entry for this command to the callers response Buffer
1048 ///
1049 CopyMem (Packet->NvmeResponse, Cq, sizeof(NVM_EXPRESS_RESPONSE));
1050
1051 if (!EFI_ERROR(Status)) { // We still need to check CQ status if no timeout error occured
1052 Status = NvmeCheckCqStatus (Cq);
1053 }
1054 NVME_SET_CQHDBL (Nvme, Qid, &Nvme->CqHdbl[Qid]);
1055
1056 EXIT:
1057 return Status;
1058 }
1059
1060 /**
1061 Get identify controller Data.
1062
1063 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
1064 @param[in] Buffer - The Buffer used to store the identify controller Data.
1065
1066 @return EFI_SUCCESS - Successfully get the identify controller Data.
1067 @return others - Fail to get the identify controller Data.
1068
1069 **/
1070 STATIC
1071 EFI_STATUS
1072 NvmeIdentifyController (
1073 IN NVME_CONTEXT *Nvme,
1074 IN VOID *Buffer
1075 )
1076 {
1077 NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
1078 NVM_EXPRESS_COMMAND Command;
1079 NVM_EXPRESS_RESPONSE Response;
1080 EFI_STATUS Status;
1081
1082 ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1083 ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1084 ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1085
1086 Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_OPC;
1087 Command.Cdw0.Cid = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1088 //
1089 // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
1090 // For the Identify command, the Namespace Identifier is only used for the Namespace Data structure.
1091 //
1092 Command.Nsid = 0;
1093
1094 CommandPacket.NvmeCmd = &Command;
1095 CommandPacket.NvmeResponse = &Response;
1096 CommandPacket.TransferBuffer = (UINT64)(UINTN)Buffer;
1097 CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA);
1098 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1099 CommandPacket.QueueId = NVME_ADMIN_QUEUE;
1100 //
1101 // Set bit 0 (Cns bit) to 1 to identify a controller
1102 //
1103 Command.Cdw10 = 1;
1104 Command.Flags = CDW10_VALID;
1105
1106 Status = NvmePassThru (
1107 Nvme,
1108 NVME_CONTROLLER_ID,
1109 0,
1110 &CommandPacket
1111 );
1112 if (!EFI_ERROR (Status)) {
1113 Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1114 }
1115
1116 return Status;
1117 }
1118
1119 /**
1120 Get specified identify namespace Data.
1121
1122 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
1123 @param[in] NamespaceId - The specified namespace identifier.
1124 @param[in] Buffer - The Buffer used to store the identify namespace Data.
1125
1126 @return EFI_SUCCESS - Successfully get the identify namespace Data.
1127 @return others - Fail to get the identify namespace Data.
1128
1129 **/
1130 STATIC
1131 EFI_STATUS
1132 NvmeIdentifyNamespace (
1133 IN NVME_CONTEXT *Nvme,
1134 IN UINT32 NamespaceId,
1135 IN VOID *Buffer
1136 )
1137 {
1138 NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
1139 NVM_EXPRESS_COMMAND Command;
1140 NVM_EXPRESS_RESPONSE Response;
1141 EFI_STATUS Status;
1142
1143 ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1144 ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1145 ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1146
1147 CommandPacket.NvmeCmd = &Command;
1148 CommandPacket.NvmeResponse = &Response;
1149
1150 Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_OPC;
1151 Command.Cdw0.Cid = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1152 Command.Nsid = NamespaceId;
1153 CommandPacket.TransferBuffer = (UINT64)(UINTN)Buffer;
1154 CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA);
1155 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1156 CommandPacket.QueueId = NVME_ADMIN_QUEUE;
1157 //
1158 // Set bit 0 (Cns bit) to 1 to identify a namespace
1159 //
1160 CommandPacket.NvmeCmd->Cdw10 = 0;
1161 CommandPacket.NvmeCmd->Flags = CDW10_VALID;
1162
1163 Status = NvmePassThru (
1164 Nvme,
1165 NamespaceId,
1166 0,
1167 &CommandPacket
1168 );
1169 if (!EFI_ERROR (Status)) {
1170 Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1171 }
1172
1173 return Status;
1174 }
1175
1176 /**
1177 Get Block Size for specific namespace of NVME.
1178
1179 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
1180
1181 @return - Block Size in bytes
1182
1183 **/
1184 STATIC
1185 UINT32
1186 NvmeGetBlockSize (
1187 IN NVME_CONTEXT *Nvme
1188 )
1189 {
1190 UINT32 BlockSize;
1191 UINT32 Lbads;
1192 UINT32 Flbas;
1193 UINT32 LbaFmtIdx;
1194
1195 Flbas = Nvme->NamespaceData->Flbas;
1196 LbaFmtIdx = Flbas & 3;
1197 Lbads = Nvme->NamespaceData->LbaFormat[LbaFmtIdx].Lbads;
1198
1199 BlockSize = (UINT32)1 << Lbads;
1200 return BlockSize;
1201 }
1202
1203 /**
1204 Get last LBA for specific namespace of NVME.
1205
1206 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
1207
1208 @return - Last LBA address
1209
1210 **/
1211 STATIC
1212 EFI_LBA
1213 NvmeGetLastLba (
1214 IN NVME_CONTEXT *Nvme
1215 )
1216 {
1217 EFI_LBA LastBlock;
1218 LastBlock = Nvme->NamespaceData->Nsze - 1;
1219 return LastBlock;
1220 }
1221
1222 /**
1223 Create io completion queue.
1224
1225 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
1226
1227 @return EFI_SUCCESS - Successfully create io completion queue.
1228 @return others - Fail to create io completion queue.
1229
1230 **/
1231 STATIC
1232 EFI_STATUS
1233 NvmeCreateIoCompletionQueue (
1234 IN NVME_CONTEXT *Nvme
1235 )
1236 {
1237 NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
1238 NVM_EXPRESS_COMMAND Command;
1239 NVM_EXPRESS_RESPONSE Response;
1240 EFI_STATUS Status;
1241 NVME_ADMIN_CRIOCQ CrIoCq;
1242
1243 ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1244 ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1245 ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1246 ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ));
1247
1248 CommandPacket.NvmeCmd = &Command;
1249 CommandPacket.NvmeResponse = &Response;
1250
1251 Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_OPC;
1252 Command.Cdw0.Cid = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1253 CommandPacket.TransferBuffer = (UINT64)(UINTN)Nvme->CqBuffer[NVME_IO_QUEUE];
1254 CommandPacket.TransferLength = EFI_PAGE_SIZE;
1255 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1256 CommandPacket.QueueId = NVME_ADMIN_QUEUE;
1257
1258 CrIoCq.Qid = NVME_IO_QUEUE;
1259 CrIoCq.Qsize = NVME_CCQ_SIZE;
1260 CrIoCq.Pc = 1;
1261 CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
1262 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1263
1264 Status = NvmePassThru (
1265 Nvme,
1266 NVME_CONTROLLER_ID,
1267 0,
1268 &CommandPacket
1269 );
1270 if (!EFI_ERROR (Status)) {
1271 Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1272 }
1273
1274 return Status;
1275 }
1276
1277 /**
1278 Create io submission queue.
1279
1280 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
1281
1282 @return EFI_SUCCESS - Successfully create io submission queue.
1283 @return others - Fail to create io submission queue.
1284
1285 **/
1286 STATIC
1287 EFI_STATUS
1288 NvmeCreateIoSubmissionQueue (
1289 IN NVME_CONTEXT *Nvme
1290 )
1291 {
1292 NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
1293 NVM_EXPRESS_COMMAND Command;
1294 NVM_EXPRESS_RESPONSE Response;
1295 EFI_STATUS Status;
1296 NVME_ADMIN_CRIOSQ CrIoSq;
1297
1298 ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1299 ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1300 ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1301 ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ));
1302
1303 CommandPacket.NvmeCmd = &Command;
1304 CommandPacket.NvmeResponse = &Response;
1305
1306 Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_OPC;
1307 Command.Cdw0.Cid = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1308 CommandPacket.TransferBuffer = (UINT64)(UINTN)Nvme->SqBuffer[NVME_IO_QUEUE];
1309 CommandPacket.TransferLength = EFI_PAGE_SIZE;
1310 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1311 CommandPacket.QueueId = NVME_ADMIN_QUEUE;
1312
1313 CrIoSq.Qid = NVME_IO_QUEUE;
1314 CrIoSq.Qsize = NVME_CSQ_SIZE;
1315 CrIoSq.Pc = 1;
1316 CrIoSq.Cqid = NVME_IO_QUEUE;
1317 CrIoSq.Qprio = 0;
1318 CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
1319 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1320
1321 Status = NvmePassThru (
1322 Nvme,
1323 NVME_CONTROLLER_ID,
1324 0,
1325 &CommandPacket
1326 );
1327 if (!EFI_ERROR (Status)) {
1328 Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1329 }
1330
1331 return Status;
1332 }
1333
1334 /**
1335 Security send and receive commands.
1336
1337 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
1338 @param[in] SendCommand - The flag to indicate the command type, TRUE for Send command and FALSE for receive command
1339 @param[in] SecurityProtocol - Security Protocol
1340 @param[in] SpSpecific - Security Protocol Specific
1341 @param[in] TransferLength - Transfer Length of Buffer (in bytes) - always a multiple of 512
1342 @param[in,out] TransferBuffer - Address of Data to transfer
1343
1344 @return EFI_SUCCESS - Successfully create io submission queue.
1345 @return others - Fail to send/receive commands.
1346
1347 **/
1348 EFI_STATUS
1349 NvmeSecuritySendReceive (
1350 IN NVME_CONTEXT *Nvme,
1351 IN BOOLEAN SendCommand,
1352 IN UINT8 SecurityProtocol,
1353 IN UINT16 SpSpecific,
1354 IN UINTN TransferLength,
1355 IN OUT VOID *TransferBuffer
1356 )
1357 {
1358 NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
1359 NVM_EXPRESS_COMMAND Command;
1360 NVM_EXPRESS_RESPONSE Response;
1361 EFI_STATUS Status;
1362 NVME_ADMIN_SECSEND SecSend;
1363 OACS *Oacs;
1364 UINT8 Opcode;
1365 VOID* *SecBuff;
1366
1367 Oacs = (OACS *)&Nvme->ControllerData->Oacs;
1368
1369 //
1370 // Verify security bit for Security Send/Receive commands
1371 //
1372 if (Oacs->Security == 0) {
1373 DEBUG ((DEBUG_ERROR, "Security command doesn't support.\n"));
1374 return EFI_NOT_READY;
1375 }
1376
1377 SecBuff = (VOID *)(UINTN) NVME_SEC_BASE (Nvme);
1378
1379 //
1380 // Actions for sending security command
1381 //
1382 if (SendCommand) {
1383 CopyMem (SecBuff, TransferBuffer, TransferLength);
1384 }
1385
1386 ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1387 ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1388 ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1389 ZeroMem (&SecSend, sizeof(NVME_ADMIN_SECSEND));
1390
1391 CommandPacket.NvmeCmd = &Command;
1392 CommandPacket.NvmeResponse = &Response;
1393
1394 Opcode = (UINT8)(SendCommand ? NVME_ADMIN_SECURITY_SEND_OPC : NVME_ADMIN_SECURITY_RECV_OPC);
1395 Command.Cdw0.Opcode = Opcode;
1396 Command.Cdw0.Cid = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1397 CommandPacket.TransferBuffer = (UINT64)(UINTN)SecBuff;
1398 CommandPacket.TransferLength = (UINT32)TransferLength;
1399 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1400 CommandPacket.QueueId = NVME_ADMIN_QUEUE;
1401
1402 SecSend.Spsp = SpSpecific;
1403 SecSend.Secp = SecurityProtocol;
1404 SecSend.Tl = (UINT32)TransferLength;
1405
1406 CopyMem (&CommandPacket.NvmeCmd->Cdw10, &SecSend, sizeof (NVME_ADMIN_SECSEND));
1407 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1408
1409 Status = NvmePassThru (
1410 Nvme,
1411 NVME_CONTROLLER_ID,
1412 0,
1413 &CommandPacket
1414 );
1415 if (!EFI_ERROR (Status)) {
1416 Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1417 }
1418
1419 //
1420 // Actions for receiving security command
1421 //
1422 if (!SendCommand) {
1423 CopyMem (TransferBuffer, SecBuff, TransferLength);
1424 }
1425
1426 return Status;
1427 }
1428
1429 /**
1430 Destroy io completion queue.
1431
1432 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
1433
1434 @return EFI_SUCCESS - Successfully destroy io completion queue.
1435 @return others - Fail to destroy io completion queue.
1436
1437 **/
1438 STATIC
1439 EFI_STATUS
1440 NvmeDestroyIoCompletionQueue (
1441 IN NVME_CONTEXT *Nvme
1442 )
1443 {
1444 NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
1445 NVM_EXPRESS_COMMAND Command;
1446 NVM_EXPRESS_RESPONSE Response;
1447 EFI_STATUS Status;
1448 NVME_ADMIN_DEIOCQ DelIoCq;
1449
1450 ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1451 ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1452 ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1453 ZeroMem (&DelIoCq, sizeof(NVME_ADMIN_DEIOCQ));
1454
1455 CommandPacket.NvmeCmd = &Command;
1456 CommandPacket.NvmeResponse = &Response;
1457
1458 Command.Cdw0.Opcode = NVME_ADMIN_DELIOCQ_OPC;
1459 Command.Cdw0.Cid = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1460 CommandPacket.TransferBuffer = (UINT64)(UINTN)Nvme->CqBuffer[NVME_IO_QUEUE];
1461 CommandPacket.TransferLength = EFI_PAGE_SIZE;
1462 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1463 CommandPacket.QueueId = NVME_ADMIN_QUEUE;
1464
1465 DelIoCq.Qid = NVME_IO_QUEUE;
1466 CopyMem (&CommandPacket.NvmeCmd->Cdw10, &DelIoCq, sizeof (NVME_ADMIN_DEIOCQ));
1467 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1468
1469 Status = NvmePassThru (
1470 Nvme,
1471 NVME_CONTROLLER_ID,
1472 0,
1473 &CommandPacket
1474 );
1475 if (!EFI_ERROR (Status)) {
1476 Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1477 }
1478
1479 return Status;
1480 }
1481
1482 /**
1483 Destroy io submission queue.
1484
1485 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
1486
1487 @return EFI_SUCCESS - Successfully destroy io submission queue.
1488 @return others - Fail to destroy io submission queue.
1489
1490 **/
1491 STATIC
1492 EFI_STATUS
1493 NvmeDestroyIoSubmissionQueue (
1494 IN NVME_CONTEXT *Nvme
1495 )
1496 {
1497 NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
1498 NVM_EXPRESS_COMMAND Command;
1499 NVM_EXPRESS_RESPONSE Response;
1500 EFI_STATUS Status;
1501 NVME_ADMIN_DEIOSQ DelIoSq;
1502
1503 ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1504 ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1505 ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1506 ZeroMem (&DelIoSq, sizeof(NVME_ADMIN_DEIOSQ));
1507
1508 CommandPacket.NvmeCmd = &Command;
1509 CommandPacket.NvmeResponse = &Response;
1510
1511 Command.Cdw0.Opcode = NVME_ADMIN_DELIOSQ_OPC;
1512 Command.Cdw0.Cid = Nvme->Cid[NVME_ADMIN_QUEUE]++;
1513 CommandPacket.TransferBuffer = (UINT64)(UINTN)Nvme->SqBuffer[NVME_IO_QUEUE];
1514 CommandPacket.TransferLength = EFI_PAGE_SIZE;
1515 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1516 CommandPacket.QueueId = NVME_ADMIN_QUEUE;
1517
1518 DelIoSq.Qid = NVME_IO_QUEUE;
1519 CopyMem (&CommandPacket.NvmeCmd->Cdw10, &DelIoSq, sizeof (NVME_ADMIN_DEIOSQ));
1520 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
1521
1522 Status = NvmePassThru (
1523 Nvme,
1524 NVME_CONTROLLER_ID,
1525 0,
1526 &CommandPacket
1527 );
1528 if (!EFI_ERROR (Status)) {
1529 Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
1530 }
1531
1532 return Status;
1533 }
1534
1535 /**
1536 Allocate transfer-related Data struct which is used at Nvme.
1537
1538 @param[in, out] Nvme The pointer to the NVME_CONTEXT Data structure.
1539
1540 @retval EFI_OUT_OF_RESOURCE No enough resource.
1541 @retval EFI_SUCCESS Successful to allocate resource.
1542
1543 **/
1544 EFI_STATUS
1545 EFIAPI
1546 NvmeAllocateResource (
1547 IN OUT NVME_CONTEXT *Nvme
1548 )
1549 {
1550 EFI_STATUS Status;
1551 EFI_PHYSICAL_ADDRESS DeviceAddress;
1552 VOID *Base;
1553 VOID *Mapping;
1554
1555 //
1556 // Allocate resources for DMA.
1557 //
1558 Status = IoMmuAllocateBuffer (
1559 EFI_SIZE_TO_PAGES (NVME_MEM_MAX_SIZE),
1560 &Base,
1561 &DeviceAddress,
1562 &Mapping
1563 );
1564 if (EFI_ERROR (Status)) {
1565 return EFI_OUT_OF_RESOURCES;
1566 }
1567 ASSERT (DeviceAddress == ((EFI_PHYSICAL_ADDRESS) (UINTN) Base));
1568 Nvme->BaseMemMapping = Mapping;
1569 Nvme->BaseMem = Base;
1570 ZeroMem (Nvme->BaseMem, EFI_PAGE_SIZE * EFI_SIZE_TO_PAGES (NVME_MEM_MAX_SIZE));
1571
1572 DEBUG ((
1573 DEBUG_INFO,
1574 "%a() NvmeContext 0x%x\n",
1575 __FUNCTION__,
1576 Nvme->BaseMem
1577 ));
1578
1579 return EFI_SUCCESS;
1580 }
1581
1582 /**
1583 Free allocated transfer-related Data struct which is used at NVMe.
1584
1585 @param[in, out] Nvme The pointer to the NVME_CONTEXT Data structure.
1586
1587 **/
1588 VOID
1589 EFIAPI
1590 NvmeFreeResource (
1591 IN OUT NVME_CONTEXT *Nvme
1592 )
1593 {
1594 if (Nvme->BaseMem != NULL) {
1595 IoMmuFreeBuffer (
1596 EFI_SIZE_TO_PAGES (NVME_MEM_MAX_SIZE),
1597 Nvme->BaseMem,
1598 Nvme->BaseMemMapping
1599 );
1600 Nvme->BaseMem = NULL;
1601 }
1602 }
1603
1604 /**
1605 Initialize the Nvm Express controller.
1606
1607 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
1608
1609 @retval EFI_SUCCESS - The NVM Express Controller is initialized successfully.
1610 @retval Others - A device error occurred while initializing the controller.
1611
1612 **/
1613 EFI_STATUS
1614 NvmeControllerInit (
1615 IN NVME_CONTEXT *Nvme
1616 )
1617 {
1618 EFI_STATUS Status;
1619 NVME_AQA Aqa;
1620 NVME_ASQ Asq;
1621 NVME_ACQ Acq;
1622 NVME_VER Ver;
1623
1624 UINT32 MlBAR;
1625 UINT32 MuBAR;
1626
1627 ///
1628 /// Update PCIE BAR0/1 for NVME device
1629 ///
1630 MlBAR = Nvme->Nbar;
1631 MuBAR = 0;
1632 PciWrite32 (Nvme->PciBase + 0x10, MlBAR); // MLBAR (BAR0)
1633 PciWrite32 (Nvme->PciBase + 0x14, MuBAR); // MUBAR (BAR1)
1634
1635 ///
1636 /// Enable PCIE decode
1637 ///
1638 PciWrite8 (Nvme->PciBase + NVME_PCIE_PCICMD, 0x6);
1639
1640 // Version
1641 NVME_GET_VER (Nvme, &Ver);
1642 if (!(Ver.Mjr == 0x0001) && (Ver.Mnr == 0x0000)) {
1643 DEBUG ((DEBUG_INFO, "\n!!!\n!!! NVME Version mismatch for the implementation !!!\n!!!\n"));
1644 }
1645
1646 ///
1647 /// Read the Controller Capabilities register and verify that the NVM command set is supported
1648 ///
1649 Status = NVME_GET_CAP (Nvme, &Nvme->Cap);
1650 if (EFI_ERROR (Status)) {
1651 DEBUG ((DEBUG_ERROR, "NVME_GET_CAP fail, Status: %r\n", Status));
1652 goto Done;
1653 }
1654
1655 if (Nvme->Cap.Css != 0x01) {
1656 DEBUG ((DEBUG_ERROR, "NvmeControllerInit fail: the controller doesn't support NVMe command set\n"));
1657 Status = EFI_UNSUPPORTED;
1658 goto Done;
1659 }
1660
1661 ///
1662 /// Currently the driver only supports 4k page Size.
1663 ///
1664 if ((Nvme->Cap.Mpsmin + 12) > EFI_PAGE_SHIFT) {
1665 DEBUG ((DEBUG_ERROR, "NvmeControllerInit fail: only supports 4k page Size\n"));
1666 ASSERT (FALSE);
1667 Status = EFI_UNSUPPORTED;
1668 goto Done;
1669 }
1670
1671 Nvme->Cid[0] = 0;
1672 Nvme->Cid[1] = 0;
1673
1674 Nvme->Pt[0] = 0;
1675 Nvme->Pt[1] = 0;
1676
1677 ZeroMem ((VOID *)(UINTN)(&(Nvme->SqTdbl[0])), sizeof (NVME_SQTDBL) * NVME_MAX_IO_QUEUES);
1678 ZeroMem ((VOID *)(UINTN)(&(Nvme->CqHdbl[0])), sizeof (NVME_CQHDBL) * NVME_MAX_IO_QUEUES);
1679
1680 ZeroMem (Nvme->BaseMem, NVME_MEM_MAX_SIZE);
1681
1682 Status = NvmeDisableController (Nvme);
1683 if (EFI_ERROR(Status)) {
1684 DEBUG ((DEBUG_ERROR, "NvmeDisableController fail, Status: %r\n", Status));
1685 goto Done;
1686 }
1687
1688 ///
1689 /// set number of entries admin submission & completion queues.
1690 ///
1691 Aqa.Asqs = NVME_ASQ_SIZE;
1692 Aqa.Rsvd1 = 0;
1693 Aqa.Acqs = NVME_ACQ_SIZE;
1694 Aqa.Rsvd2 = 0;
1695
1696 ///
1697 /// Address of admin submission queue.
1698 ///
1699 Asq = (UINT64)(UINTN)(NVME_ASQ_BASE (Nvme) & ~0xFFF);
1700
1701 ///
1702 /// Address of admin completion queue.
1703 ///
1704 Acq = (UINT64)(UINTN)(NVME_ACQ_BASE (Nvme) & ~0xFFF);
1705
1706 ///
1707 /// Address of I/O submission & completion queue.
1708 ///
1709 Nvme->SqBuffer[0] = (NVME_SQ *)(UINTN)NVME_ASQ_BASE (Nvme); // NVME_ADMIN_QUEUE
1710 Nvme->CqBuffer[0] = (NVME_CQ *)(UINTN)NVME_ACQ_BASE (Nvme); // NVME_ADMIN_QUEUE
1711 Nvme->SqBuffer[1] = (NVME_SQ *)(UINTN)NVME_SQ_BASE (Nvme, 0); // NVME_IO_QUEUE
1712 Nvme->CqBuffer[1] = (NVME_CQ *)(UINTN)NVME_CQ_BASE (Nvme, 0); // NVME_IO_QUEUE
1713
1714 DEBUG ((DEBUG_INFO, "Admin Submission Queue Size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));
1715 DEBUG ((DEBUG_INFO, "Admin Completion Queue Size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));
1716 DEBUG ((DEBUG_INFO, "Admin Submission Queue (SqBuffer[0]) = [%08X]\n", Nvme->SqBuffer[0]));
1717 DEBUG ((DEBUG_INFO, "Admin Completion Queue (CqBuffer[0]) = [%08X]\n", Nvme->CqBuffer[0]));
1718 DEBUG ((DEBUG_INFO, "I/O Submission Queue (SqBuffer[1]) = [%08X]\n", Nvme->SqBuffer[1]));
1719 DEBUG ((DEBUG_INFO, "I/O Completion Queue (CqBuffer[1]) = [%08X]\n", Nvme->CqBuffer[1]));
1720
1721 ///
1722 /// Program admin queue attributes.
1723 ///
1724 Status = NVME_SET_AQA (Nvme, &Aqa);
1725 if (EFI_ERROR(Status)) {
1726 goto Done;
1727 }
1728
1729 ///
1730 /// Program admin submission queue address.
1731 ///
1732 Status = NVME_SET_ASQ (Nvme, &Asq);
1733 if (EFI_ERROR(Status)) {
1734 goto Done;
1735 }
1736
1737 ///
1738 /// Program admin completion queue address.
1739 ///
1740 Status = NVME_SET_ACQ (Nvme, &Acq);
1741 if (EFI_ERROR(Status)) {
1742 goto Done;
1743 }
1744
1745 Status = NvmeEnableController (Nvme);
1746 if (EFI_ERROR(Status)) {
1747 goto Done;
1748 }
1749
1750 ///
1751 /// Create one I/O completion queue.
1752 ///
1753 Status = NvmeCreateIoCompletionQueue (Nvme);
1754 if (EFI_ERROR(Status)) {
1755 goto Done;
1756 }
1757
1758 ///
1759 /// Create one I/O Submission queue.
1760 ///
1761 Status = NvmeCreateIoSubmissionQueue (Nvme);
1762 if (EFI_ERROR(Status)) {
1763 goto Done;
1764 }
1765
1766 ///
1767 /// Get current Identify Controller Data
1768 ///
1769 Nvme->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)(UINTN) NVME_CONTROL_DATA_BASE (Nvme);
1770 Status = NvmeIdentifyController (Nvme, Nvme->ControllerData);
1771 if (EFI_ERROR(Status)) {
1772 goto Done;
1773 }
1774
1775 ///
1776 /// Dump NvmExpress Identify Controller Data
1777 ///
1778 Nvme->ControllerData->Sn[19] = 0;
1779 Nvme->ControllerData->Mn[39] = 0;
1780 //NvmeDumpIdentifyController (Nvme->ControllerData);
1781
1782 ///
1783 /// Get current Identify Namespace Data
1784 ///
1785 Nvme->NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)NVME_NAMESPACE_DATA_BASE (Nvme);
1786 Status = NvmeIdentifyNamespace (Nvme, Nvme->Nsid, Nvme->NamespaceData);
1787 if (EFI_ERROR(Status)) {
1788 DEBUG ((DEBUG_ERROR, "NvmeIdentifyNamespace fail, Status = %r\n", Status));
1789 goto Done;
1790 }
1791
1792 ///
1793 /// Dump NvmExpress Identify Namespace Data
1794 ///
1795 if (Nvme->NamespaceData->Ncap == 0) {
1796 DEBUG ((DEBUG_ERROR, "Invalid Namespace, Ncap: %lx\n", Nvme->NamespaceData->Ncap));
1797 Status = EFI_DEVICE_ERROR;
1798 goto Done;
1799 }
1800
1801 Nvme->BlockSize = NvmeGetBlockSize (Nvme);
1802 Nvme->LastBlock = NvmeGetLastLba (Nvme);
1803
1804 Nvme->State = NvmeStatusInit;
1805
1806 return EFI_SUCCESS;
1807
1808 Done:
1809 return Status;
1810 }
1811
1812 /**
1813 Un-initialize the Nvm Express controller.
1814
1815 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
1816
1817 @retval EFI_SUCCESS - The NVM Express Controller is un-initialized successfully.
1818 @retval Others - A device error occurred while un-initializing the controller.
1819
1820 **/
1821 EFI_STATUS
1822 NvmeControllerExit (
1823 IN NVME_CONTEXT *Nvme
1824 )
1825 {
1826 EFI_STATUS Status;
1827
1828 Status = EFI_SUCCESS;
1829 if (Nvme->State == NvmeStatusInit || Nvme->State == NvmeStatusMax) {
1830 ///
1831 /// Destroy I/O Submission queue.
1832 ///
1833 Status = NvmeDestroyIoSubmissionQueue (Nvme);
1834 if (EFI_ERROR(Status)) {
1835 DEBUG ((DEBUG_ERROR, "NvmeDestroyIoSubmissionQueue fail, Status = %r\n", Status));
1836 return Status;
1837 }
1838
1839 ///
1840 /// Destroy I/O completion queue.
1841 ///
1842 Status = NvmeDestroyIoCompletionQueue (Nvme);
1843 if (EFI_ERROR(Status)) {
1844 DEBUG ((DEBUG_ERROR, "NvmeDestroyIoCompletionQueue fail, Status = %r\n", Status));
1845 return Status;
1846 }
1847
1848 Status = NvmeShutdownController (Nvme);
1849 if (EFI_ERROR(Status)) {
1850 DEBUG ((DEBUG_ERROR, "NvmeShutdownController fail, Status: %r\n", Status));
1851 }
1852 }
1853
1854 ///
1855 /// Disable PCIE decode
1856 ///
1857 PciWrite8 (Nvme->PciBase + NVME_PCIE_PCICMD, 0x0);
1858 PciWrite32 (Nvme->PciBase + 0x10, 0); // MLBAR (BAR0)
1859 PciWrite32 (Nvme->PciBase + 0x14, 0); // MUBAR (BAR1)
1860
1861 Nvme->State = NvmeStatusUnknown;
1862 return Status;
1863 }
1864
1865 /**
1866 Read sector Data from the NVMe device.
1867
1868 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
1869 @param[in,out] Buffer - The Buffer used to store the Data read from the device.
1870 @param[in] Lba - The start block number.
1871 @param[in] Blocks - Total block number to be read.
1872
1873 @retval EFI_SUCCESS - Datum are read from the device.
1874 @retval Others - Fail to read all the datum.
1875
1876 **/
1877 EFI_STATUS
1878 NvmeReadSectors (
1879 IN NVME_CONTEXT *Nvme,
1880 IN OUT UINT64 Buffer,
1881 IN UINT64 Lba,
1882 IN UINT32 Blocks
1883 )
1884 {
1885 UINT32 Bytes;
1886 NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
1887 NVM_EXPRESS_COMMAND Command;
1888 NVM_EXPRESS_RESPONSE Response;
1889 EFI_STATUS Status;
1890 UINT32 BlockSize;
1891
1892 BlockSize = Nvme->BlockSize;
1893 Bytes = Blocks * BlockSize;
1894
1895 ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1896 ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1897 ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1898
1899 CommandPacket.NvmeCmd = &Command;
1900 CommandPacket.NvmeResponse = &Response;
1901
1902 CommandPacket.NvmeCmd->Cdw0.Opcode = NVME_IO_READ_OPC;
1903 CommandPacket.NvmeCmd->Cdw0.Cid = Nvme->Cid[NVME_IO_QUEUE]++;
1904 CommandPacket.NvmeCmd->Nsid = Nvme->Nsid;
1905 CommandPacket.TransferBuffer = Buffer;
1906
1907 CommandPacket.TransferLength = Bytes;
1908 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1909 CommandPacket.QueueId = NVME_IO_QUEUE;
1910
1911 CommandPacket.NvmeCmd->Cdw10 = (UINT32)Lba;
1912 CommandPacket.NvmeCmd->Cdw11 = (UINT32)(RShiftU64 (Lba, 32));
1913 CommandPacket.NvmeCmd->Cdw12 = (Blocks - 1) & 0xFFFF;
1914
1915 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID | CDW12_VALID;
1916
1917 Status = NvmePassThru (
1918 Nvme,
1919 Nvme->Nsid,
1920 0,
1921 &CommandPacket
1922 );
1923
1924 return Status;
1925 }
1926
1927 /**
1928 Write sector Data to the NVMe device.
1929
1930 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
1931 @param[in] Buffer - The Buffer to be written into the device.
1932 @param[in] Lba - The start block number.
1933 @param[in] Blocks - Total block number to be written.
1934
1935 @retval EFI_SUCCESS - Datum are written into the Buffer.
1936 @retval Others - Fail to write all the datum.
1937
1938 **/
1939 EFI_STATUS
1940 NvmeWriteSectors (
1941 IN NVME_CONTEXT *Nvme,
1942 IN UINT64 Buffer,
1943 IN UINT64 Lba,
1944 IN UINT32 Blocks
1945 )
1946 {
1947 NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
1948 NVM_EXPRESS_COMMAND Command;
1949 NVM_EXPRESS_RESPONSE Response;
1950 EFI_STATUS Status;
1951 UINT32 Bytes;
1952 UINT32 BlockSize;
1953
1954 BlockSize = Nvme->BlockSize;
1955 Bytes = Blocks * BlockSize;
1956
1957 ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
1958 ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
1959 ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
1960
1961 CommandPacket.NvmeCmd = &Command;
1962 CommandPacket.NvmeResponse = &Response;
1963
1964 CommandPacket.NvmeCmd->Cdw0.Opcode = NVME_IO_WRITE_OPC;
1965 CommandPacket.NvmeCmd->Cdw0.Cid = Nvme->Cid[NVME_IO_QUEUE]++;
1966 CommandPacket.NvmeCmd->Nsid = Nvme->Nsid;
1967 CommandPacket.TransferBuffer = Buffer;
1968
1969 CommandPacket.TransferLength = Bytes;
1970 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
1971 CommandPacket.QueueId = NVME_IO_QUEUE;
1972
1973 CommandPacket.NvmeCmd->Cdw10 = (UINT32)Lba;
1974 CommandPacket.NvmeCmd->Cdw11 = (UINT32)(RShiftU64 (Lba, 32));
1975 CommandPacket.NvmeCmd->Cdw12 = (Blocks - 1) & 0xFFFF;
1976
1977 CommandPacket.MetadataBuffer = (UINT64)(UINTN)NULL;
1978 CommandPacket.MetadataLength = 0;
1979
1980 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID | CDW12_VALID;
1981
1982 Status = NvmePassThru (
1983 Nvme,
1984 Nvme->Nsid,
1985 0,
1986 &CommandPacket
1987 );
1988
1989 return Status;
1990 }
1991
1992 /**
1993 Flushes all modified Data to the device.
1994
1995 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
1996
1997 @retval EFI_SUCCESS - Datum are written into the Buffer.
1998 @retval Others - Fail to write all the datum.
1999
2000 **/
2001 EFI_STATUS
2002 NvmeFlush (
2003 IN NVME_CONTEXT *Nvme
2004 )
2005 {
2006 NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
2007 NVM_EXPRESS_COMMAND Command;
2008 NVM_EXPRESS_RESPONSE Response;
2009 EFI_STATUS Status;
2010
2011 ZeroMem (&CommandPacket, sizeof(NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
2012 ZeroMem (&Command, sizeof(NVM_EXPRESS_COMMAND));
2013 ZeroMem (&Response, sizeof(NVM_EXPRESS_RESPONSE));
2014
2015 CommandPacket.NvmeCmd = &Command;
2016 CommandPacket.NvmeResponse = &Response;
2017
2018 CommandPacket.NvmeCmd->Cdw0.Opcode = NVME_IO_FLUSH_OPC;
2019 CommandPacket.NvmeCmd->Cdw0.Cid = Nvme->Cid[NVME_IO_QUEUE]++;
2020 CommandPacket.NvmeCmd->Nsid = Nvme->Nsid;
2021 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
2022 CommandPacket.QueueId = NVME_IO_QUEUE;
2023
2024 Status = NvmePassThru (
2025 Nvme,
2026 Nvme->Nsid,
2027 0,
2028 &CommandPacket
2029 );
2030 if (!EFI_ERROR (Status)) {
2031 Status = NvmeWaitAllComplete (Nvme, CommandPacket.QueueId);
2032 }
2033
2034 return Status;
2035 }
2036
2037 /**
2038 Read some blocks from the device.
2039
2040 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
2041 @param[out] Buffer - The Buffer used to store the Data read from the device.
2042 @param[in] Lba - The start block number.
2043 @param[in] Blocks - Total block number to be read.
2044
2045 @retval EFI_SUCCESS - Datum are read from the device.
2046 @retval Others - Fail to read all the datum.
2047
2048 **/
2049 EFI_STATUS
2050 NvmeRead (
2051 IN NVME_CONTEXT *Nvme,
2052 OUT UINT64 Buffer,
2053 IN UINT64 Lba,
2054 IN UINTN Blocks
2055 )
2056 {
2057 EFI_STATUS Status;
2058 UINT32 BlockSize;
2059 UINT32 MaxTransferBlocks;
2060
2061 ASSERT (Blocks <= NVME_MAX_SECTORS);
2062 Status = EFI_SUCCESS;
2063 BlockSize = Nvme->BlockSize;
2064 if (Nvme->ControllerData->Mdts != 0) {
2065 MaxTransferBlocks = (1 << (Nvme->ControllerData->Mdts)) * (1 << (Nvme->Cap.Mpsmin + 12)) / BlockSize;
2066 } else {
2067 MaxTransferBlocks = 1024;
2068 }
2069
2070 while (Blocks > 0) {
2071 if (Blocks > MaxTransferBlocks) {
2072 Status = NvmeReadSectors (Nvme, Buffer, Lba, MaxTransferBlocks);
2073
2074 Blocks -= MaxTransferBlocks;
2075 Buffer += (MaxTransferBlocks * BlockSize);
2076 Lba += MaxTransferBlocks;
2077 } else {
2078 Status = NvmeReadSectors (Nvme, Buffer, Lba, (UINT32) Blocks);
2079 Blocks = 0;
2080 }
2081
2082 if (EFI_ERROR(Status)) {
2083 DEBUG ((DEBUG_ERROR, "NvmeRead fail, Status = %r\n", Status));
2084 break;
2085 }
2086 }
2087
2088 return Status;
2089 }
2090
2091 /**
2092 Write some blocks to the device.
2093
2094 @param[in] Nvme - The pointer to the NVME_CONTEXT Data structure.
2095 @param[in] Buffer - The Buffer to be written into the device.
2096 @param[in] Lba - The start block number.
2097 @param[in] Blocks - Total block number to be written.
2098
2099 @retval EFI_SUCCESS - Datum are written into the Buffer.
2100 @retval Others - Fail to write all the datum.
2101
2102 **/
2103 EFI_STATUS
2104 NvmeWrite (
2105 IN NVME_CONTEXT *Nvme,
2106 IN UINT64 Buffer,
2107 IN UINT64 Lba,
2108 IN UINTN Blocks
2109 )
2110 {
2111 EFI_STATUS Status;
2112 UINT32 BlockSize;
2113 UINT32 MaxTransferBlocks;
2114
2115 ASSERT (Blocks <= NVME_MAX_SECTORS);
2116 Status = EFI_SUCCESS;
2117 BlockSize = Nvme->BlockSize;
2118
2119 if (Nvme->ControllerData->Mdts != 0) {
2120 MaxTransferBlocks = (1 << (Nvme->ControllerData->Mdts)) * (1 << (Nvme->Cap.Mpsmin + 12)) / BlockSize;
2121 } else {
2122 MaxTransferBlocks = 1024;
2123 }
2124
2125 while (Blocks > 0) {
2126 if (Blocks > MaxTransferBlocks) {
2127 Status = NvmeWriteSectors (Nvme, Buffer, Lba, MaxTransferBlocks);
2128
2129 Blocks -= MaxTransferBlocks;
2130 Buffer += (MaxTransferBlocks * BlockSize);
2131 Lba += MaxTransferBlocks;
2132 } else {
2133 Status = NvmeWriteSectors (Nvme, Buffer, Lba, (UINT32) Blocks);
2134 Blocks = 0;
2135 }
2136
2137 if (EFI_ERROR(Status)) {
2138 DEBUG ((DEBUG_ERROR, "NvmeWrite fail, Status = %r\n", Status));
2139 break;
2140 }
2141 }
2142
2143 return Status;
2144 }
EFI Development Kit II mirror for PVE