]> git.proxmox.com Git - mirror_edk2.git/blob - MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c
MdeModulePkg/NvmExpressDxe: fix error status override
[mirror_edk2.git] / MdeModulePkg / Bus / Pci / NvmExpressDxe / NvmExpressPassthru.c
1 /** @file
2 NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
3 NVM Express specification.
4
5 (C) Copyright 2014 Hewlett-Packard Development Company, L.P.<BR>
6 Copyright (c) 2013 - 2017, Intel Corporation. All rights reserved.<BR>
7 This program and the accompanying materials
8 are licensed and made available under the terms and conditions of the BSD License
9 which accompanies this distribution. The full text of the license may be found at
10 http://opensource.org/licenses/bsd-license.php.
11
12 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
13 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
14
15 **/
16
17 #include "NvmExpress.h"
18
19 /**
20 Dump the execution status from a given completion queue entry.
21
22 @param[in] Cq A pointer to the NVME_CQ item.
23
24 **/
25 VOID
26 NvmeDumpStatus (
27 IN NVME_CQ *Cq
28 )
29 {
30 DEBUG ((EFI_D_VERBOSE, "Dump NVMe Completion Entry Status from [0x%x]:\n", Cq));
31
32 DEBUG ((EFI_D_VERBOSE, " SQ Identifier : [0x%x], Phase Tag : [%d], Cmd Identifier : [0x%x]\n", Cq->Sqid, Cq->Pt, Cq->Cid));
33
34 DEBUG ((EFI_D_VERBOSE, " NVMe Cmd Execution Result - "));
35
36 switch (Cq->Sct) {
37 case 0x0:
38 switch (Cq->Sc) {
39 case 0x0:
40 DEBUG ((EFI_D_VERBOSE, "Successful Completion\n"));
41 break;
42 case 0x1:
43 DEBUG ((EFI_D_VERBOSE, "Invalid Command Opcode\n"));
44 break;
45 case 0x2:
46 DEBUG ((EFI_D_VERBOSE, "Invalid Field in Command\n"));
47 break;
48 case 0x3:
49 DEBUG ((EFI_D_VERBOSE, "Command ID Conflict\n"));
50 break;
51 case 0x4:
52 DEBUG ((EFI_D_VERBOSE, "Data Transfer Error\n"));
53 break;
54 case 0x5:
55 DEBUG ((EFI_D_VERBOSE, "Commands Aborted due to Power Loss Notification\n"));
56 break;
57 case 0x6:
58 DEBUG ((EFI_D_VERBOSE, "Internal Device Error\n"));
59 break;
60 case 0x7:
61 DEBUG ((EFI_D_VERBOSE, "Command Abort Requested\n"));
62 break;
63 case 0x8:
64 DEBUG ((EFI_D_VERBOSE, "Command Aborted due to SQ Deletion\n"));
65 break;
66 case 0x9:
67 DEBUG ((EFI_D_VERBOSE, "Command Aborted due to Failed Fused Command\n"));
68 break;
69 case 0xA:
70 DEBUG ((EFI_D_VERBOSE, "Command Aborted due to Missing Fused Command\n"));
71 break;
72 case 0xB:
73 DEBUG ((EFI_D_VERBOSE, "Invalid Namespace or Format\n"));
74 break;
75 case 0xC:
76 DEBUG ((EFI_D_VERBOSE, "Command Sequence Error\n"));
77 break;
78 case 0xD:
79 DEBUG ((EFI_D_VERBOSE, "Invalid SGL Last Segment Descriptor\n"));
80 break;
81 case 0xE:
82 DEBUG ((EFI_D_VERBOSE, "Invalid Number of SGL Descriptors\n"));
83 break;
84 case 0xF:
85 DEBUG ((EFI_D_VERBOSE, "Data SGL Length Invalid\n"));
86 break;
87 case 0x10:
88 DEBUG ((EFI_D_VERBOSE, "Metadata SGL Length Invalid\n"));
89 break;
90 case 0x11:
91 DEBUG ((EFI_D_VERBOSE, "SGL Descriptor Type Invalid\n"));
92 break;
93 case 0x80:
94 DEBUG ((EFI_D_VERBOSE, "LBA Out of Range\n"));
95 break;
96 case 0x81:
97 DEBUG ((EFI_D_VERBOSE, "Capacity Exceeded\n"));
98 break;
99 case 0x82:
100 DEBUG ((EFI_D_VERBOSE, "Namespace Not Ready\n"));
101 break;
102 case 0x83:
103 DEBUG ((EFI_D_VERBOSE, "Reservation Conflict\n"));
104 break;
105 }
106 break;
107
108 case 0x1:
109 switch (Cq->Sc) {
110 case 0x0:
111 DEBUG ((EFI_D_VERBOSE, "Completion Queue Invalid\n"));
112 break;
113 case 0x1:
114 DEBUG ((EFI_D_VERBOSE, "Invalid Queue Identifier\n"));
115 break;
116 case 0x2:
117 DEBUG ((EFI_D_VERBOSE, "Maximum Queue Size Exceeded\n"));
118 break;
119 case 0x3:
120 DEBUG ((EFI_D_VERBOSE, "Abort Command Limit Exceeded\n"));
121 break;
122 case 0x5:
123 DEBUG ((EFI_D_VERBOSE, "Asynchronous Event Request Limit Exceeded\n"));
124 break;
125 case 0x6:
126 DEBUG ((EFI_D_VERBOSE, "Invalid Firmware Slot\n"));
127 break;
128 case 0x7:
129 DEBUG ((EFI_D_VERBOSE, "Invalid Firmware Image\n"));
130 break;
131 case 0x8:
132 DEBUG ((EFI_D_VERBOSE, "Invalid Interrupt Vector\n"));
133 break;
134 case 0x9:
135 DEBUG ((EFI_D_VERBOSE, "Invalid Log Page\n"));
136 break;
137 case 0xA:
138 DEBUG ((EFI_D_VERBOSE, "Invalid Format\n"));
139 break;
140 case 0xB:
141 DEBUG ((EFI_D_VERBOSE, "Firmware Application Requires Conventional Reset\n"));
142 break;
143 case 0xC:
144 DEBUG ((EFI_D_VERBOSE, "Invalid Queue Deletion\n"));
145 break;
146 case 0xD:
147 DEBUG ((EFI_D_VERBOSE, "Feature Identifier Not Saveable\n"));
148 break;
149 case 0xE:
150 DEBUG ((EFI_D_VERBOSE, "Feature Not Changeable\n"));
151 break;
152 case 0xF:
153 DEBUG ((EFI_D_VERBOSE, "Feature Not Namespace Specific\n"));
154 break;
155 case 0x10:
156 DEBUG ((EFI_D_VERBOSE, "Firmware Application Requires NVM Subsystem Reset\n"));
157 break;
158 case 0x80:
159 DEBUG ((EFI_D_VERBOSE, "Conflicting Attributes\n"));
160 break;
161 case 0x81:
162 DEBUG ((EFI_D_VERBOSE, "Invalid Protection Information\n"));
163 break;
164 case 0x82:
165 DEBUG ((EFI_D_VERBOSE, "Attempted Write to Read Only Range\n"));
166 break;
167 }
168 break;
169
170 case 0x2:
171 switch (Cq->Sc) {
172 case 0x80:
173 DEBUG ((EFI_D_VERBOSE, "Write Fault\n"));
174 break;
175 case 0x81:
176 DEBUG ((EFI_D_VERBOSE, "Unrecovered Read Error\n"));
177 break;
178 case 0x82:
179 DEBUG ((EFI_D_VERBOSE, "End-to-end Guard Check Error\n"));
180 break;
181 case 0x83:
182 DEBUG ((EFI_D_VERBOSE, "End-to-end Application Tag Check Error\n"));
183 break;
184 case 0x84:
185 DEBUG ((EFI_D_VERBOSE, "End-to-end Reference Tag Check Error\n"));
186 break;
187 case 0x85:
188 DEBUG ((EFI_D_VERBOSE, "Compare Failure\n"));
189 break;
190 case 0x86:
191 DEBUG ((EFI_D_VERBOSE, "Access Denied\n"));
192 break;
193 }
194 break;
195
196 default:
197 break;
198 }
199 }
200
201 /**
202 Create PRP lists for data transfer which is larger than 2 memory pages.
203 Note here we calcuate the number of required PRP lists and allocate them at one time.
204
205 @param[in] PciIo A pointer to the EFI_PCI_IO_PROTOCOL instance.
206 @param[in] PhysicalAddr The physical base address of data buffer.
207 @param[in] Pages The number of pages to be transfered.
208 @param[out] PrpListHost The host base address of PRP lists.
209 @param[in,out] PrpListNo The number of PRP List.
210 @param[out] Mapping The mapping value returned from PciIo.Map().
211
212 @retval The pointer to the first PRP List of the PRP lists.
213
214 **/
215 VOID*
216 NvmeCreatePrpList (
217 IN EFI_PCI_IO_PROTOCOL *PciIo,
218 IN EFI_PHYSICAL_ADDRESS PhysicalAddr,
219 IN UINTN Pages,
220 OUT VOID **PrpListHost,
221 IN OUT UINTN *PrpListNo,
222 OUT VOID **Mapping
223 )
224 {
225 UINTN PrpEntryNo;
226 UINT64 PrpListBase;
227 UINTN PrpListIndex;
228 UINTN PrpEntryIndex;
229 UINT64 Remainder;
230 EFI_PHYSICAL_ADDRESS PrpListPhyAddr;
231 UINTN Bytes;
232 EFI_STATUS Status;
233
234 //
235 // The number of Prp Entry in a memory page.
236 //
237 PrpEntryNo = EFI_PAGE_SIZE / sizeof (UINT64);
238
239 //
240 // Calculate total PrpList number.
241 //
242 *PrpListNo = (UINTN)DivU64x64Remainder ((UINT64)Pages, (UINT64)PrpEntryNo - 1, &Remainder);
243 if (*PrpListNo == 0) {
244 *PrpListNo = 1;
245 } else if ((Remainder != 0) && (Remainder != 1)) {
246 *PrpListNo += 1;
247 } else if (Remainder == 1) {
248 Remainder = PrpEntryNo;
249 } else if (Remainder == 0) {
250 Remainder = PrpEntryNo - 1;
251 }
252
253 Status = PciIo->AllocateBuffer (
254 PciIo,
255 AllocateAnyPages,
256 EfiBootServicesData,
257 *PrpListNo,
258 PrpListHost,
259 0
260 );
261
262 if (EFI_ERROR (Status)) {
263 return NULL;
264 }
265
266 Bytes = EFI_PAGES_TO_SIZE (*PrpListNo);
267 Status = PciIo->Map (
268 PciIo,
269 EfiPciIoOperationBusMasterCommonBuffer,
270 *PrpListHost,
271 &Bytes,
272 &PrpListPhyAddr,
273 Mapping
274 );
275
276 if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (*PrpListNo))) {
277 DEBUG ((EFI_D_ERROR, "NvmeCreatePrpList: create PrpList failure!\n"));
278 goto EXIT;
279 }
280 //
281 // Fill all PRP lists except of last one.
282 //
283 ZeroMem (*PrpListHost, Bytes);
284 for (PrpListIndex = 0; PrpListIndex < *PrpListNo - 1; ++PrpListIndex) {
285 PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
286
287 for (PrpEntryIndex = 0; PrpEntryIndex < PrpEntryNo; ++PrpEntryIndex) {
288 if (PrpEntryIndex != PrpEntryNo - 1) {
289 //
290 // Fill all PRP entries except of last one.
291 //
292 *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PhysicalAddr;
293 PhysicalAddr += EFI_PAGE_SIZE;
294 } else {
295 //
296 // Fill last PRP entries with next PRP List pointer.
297 //
298 *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PrpListPhyAddr + (PrpListIndex + 1) * EFI_PAGE_SIZE;
299 }
300 }
301 }
302 //
303 // Fill last PRP list.
304 //
305 PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
306 for (PrpEntryIndex = 0; PrpEntryIndex < Remainder; ++PrpEntryIndex) {
307 *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PhysicalAddr;
308 PhysicalAddr += EFI_PAGE_SIZE;
309 }
310
311 return (VOID*)(UINTN)PrpListPhyAddr;
312
313 EXIT:
314 PciIo->FreeBuffer (PciIo, *PrpListNo, *PrpListHost);
315 return NULL;
316 }
317
318
319 /**
320 Aborts the asynchronous PassThru requests.
321
322 @param[in] Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA
323 data structure.
324
325 @retval EFI_SUCCESS The asynchronous PassThru requests have been aborted.
326 @return EFI_DEVICE_ERROR Fail to abort all the asynchronous PassThru requests.
327
328 **/
329 EFI_STATUS
330 AbortAsyncPassThruTasks (
331 IN NVME_CONTROLLER_PRIVATE_DATA *Private
332 )
333 {
334 EFI_PCI_IO_PROTOCOL *PciIo;
335 LIST_ENTRY *Link;
336 LIST_ENTRY *NextLink;
337 NVME_BLKIO2_SUBTASK *Subtask;
338 NVME_BLKIO2_REQUEST *BlkIo2Request;
339 NVME_PASS_THRU_ASYNC_REQ *AsyncRequest;
340 EFI_BLOCK_IO2_TOKEN *Token;
341 EFI_TPL OldTpl;
342 EFI_STATUS Status;
343
344 PciIo = Private->PciIo;
345 OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
346
347 //
348 // Cancel the unsubmitted subtasks.
349 //
350 for (Link = GetFirstNode (&Private->UnsubmittedSubtasks);
351 !IsNull (&Private->UnsubmittedSubtasks, Link);
352 Link = NextLink) {
353 NextLink = GetNextNode (&Private->UnsubmittedSubtasks, Link);
354 Subtask = NVME_BLKIO2_SUBTASK_FROM_LINK (Link);
355 BlkIo2Request = Subtask->BlockIo2Request;
356 Token = BlkIo2Request->Token;
357
358 BlkIo2Request->UnsubmittedSubtaskNum--;
359 if (Subtask->IsLast) {
360 BlkIo2Request->LastSubtaskSubmitted = TRUE;
361 }
362 Token->TransactionStatus = EFI_ABORTED;
363
364 RemoveEntryList (Link);
365 InsertTailList (&BlkIo2Request->SubtasksQueue, Link);
366 gBS->SignalEvent (Subtask->Event);
367 }
368
369 //
370 // Cleanup the resources for the asynchronous PassThru requests.
371 //
372 for (Link = GetFirstNode (&Private->AsyncPassThruQueue);
373 !IsNull (&Private->AsyncPassThruQueue, Link);
374 Link = NextLink) {
375 NextLink = GetNextNode (&Private->AsyncPassThruQueue, Link);
376 AsyncRequest = NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link);
377
378 if (AsyncRequest->MapData != NULL) {
379 PciIo->Unmap (PciIo, AsyncRequest->MapData);
380 }
381 if (AsyncRequest->MapMeta != NULL) {
382 PciIo->Unmap (PciIo, AsyncRequest->MapMeta);
383 }
384 if (AsyncRequest->MapPrpList != NULL) {
385 PciIo->Unmap (PciIo, AsyncRequest->MapPrpList);
386 }
387 if (AsyncRequest->PrpListHost != NULL) {
388 PciIo->FreeBuffer (
389 PciIo,
390 AsyncRequest->PrpListNo,
391 AsyncRequest->PrpListHost
392 );
393 }
394
395 RemoveEntryList (Link);
396 gBS->SignalEvent (AsyncRequest->CallerEvent);
397 FreePool (AsyncRequest);
398 }
399
400 if (IsListEmpty (&Private->AsyncPassThruQueue) &&
401 IsListEmpty (&Private->UnsubmittedSubtasks)) {
402 Status = EFI_SUCCESS;
403 } else {
404 Status = EFI_DEVICE_ERROR;
405 }
406
407 gBS->RestoreTPL (OldTpl);
408
409 return Status;
410 }
411
412
413 /**
414 Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function supports
415 both blocking I/O and non-blocking I/O. The blocking I/O functionality is required, and the non-blocking
416 I/O functionality is optional.
417
418
419 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
420 @param[in] NamespaceId A 32 bit namespace ID as defined in the NVMe specification to which the NVM Express Command
421 Packet will be sent. A value of 0 denotes the NVM Express controller, a value of all 0xFF's
422 (all bytes are 0xFF) in the namespace ID specifies that the command packet should be sent to
423 all valid namespaces.
424 @param[in,out] Packet A pointer to the NVM Express Command Packet.
425 @param[in] Event If non-blocking I/O is not supported then Event is ignored, and blocking I/O is performed.
426 If Event is NULL, then blocking I/O is performed. If Event is not NULL and non-blocking I/O
427 is supported, then non-blocking I/O is performed, and Event will be signaled when the NVM
428 Express Command Packet completes.
429
430 @retval EFI_SUCCESS The NVM Express Command Packet was sent by the host. TransferLength bytes were transferred
431 to, or from DataBuffer.
432 @retval EFI_BAD_BUFFER_SIZE The NVM Express Command Packet was not executed. The number of bytes that could be transferred
433 is returned in TransferLength.
434 @retval EFI_NOT_READY The NVM Express Command Packet could not be sent because the controller is not ready. The caller
435 may retry again later.
436 @retval EFI_DEVICE_ERROR A device error occurred while attempting to send the NVM Express Command Packet.
437 @retval EFI_INVALID_PARAMETER NamespaceId or the contents of EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET are invalid. The NVM
438 Express Command Packet was not sent, so no additional status information is available.
439 @retval EFI_UNSUPPORTED The command described by the NVM Express Command Packet is not supported by the NVM Express
440 controller. The NVM Express Command Packet was not sent so no additional status information
441 is available.
442 @retval EFI_TIMEOUT A timeout occurred while waiting for the NVM Express Command Packet to execute.
443
444 **/
445 EFI_STATUS
446 EFIAPI
447 NvmExpressPassThru (
448 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *This,
449 IN UINT32 NamespaceId,
450 IN OUT EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *Packet,
451 IN EFI_EVENT Event OPTIONAL
452 )
453 {
454 NVME_CONTROLLER_PRIVATE_DATA *Private;
455 EFI_STATUS Status;
456 EFI_STATUS PreviousStatus;
457 EFI_PCI_IO_PROTOCOL *PciIo;
458 NVME_SQ *Sq;
459 NVME_CQ *Cq;
460 UINT16 QueueId;
461 UINT32 Bytes;
462 UINT16 Offset;
463 EFI_EVENT TimerEvent;
464 EFI_PCI_IO_PROTOCOL_OPERATION Flag;
465 EFI_PHYSICAL_ADDRESS PhyAddr;
466 VOID *MapData;
467 VOID *MapMeta;
468 VOID *MapPrpList;
469 UINTN MapLength;
470 UINT64 *Prp;
471 VOID *PrpListHost;
472 UINTN PrpListNo;
473 UINT32 Attributes;
474 UINT32 IoAlign;
475 UINT32 MaxTransLen;
476 UINT32 Data;
477 NVME_PASS_THRU_ASYNC_REQ *AsyncRequest;
478 EFI_TPL OldTpl;
479
480 //
481 // check the data fields in Packet parameter.
482 //
483 if ((This == NULL) || (Packet == NULL)) {
484 return EFI_INVALID_PARAMETER;
485 }
486
487 if ((Packet->NvmeCmd == NULL) || (Packet->NvmeCompletion == NULL)) {
488 return EFI_INVALID_PARAMETER;
489 }
490
491 if (Packet->QueueType != NVME_ADMIN_QUEUE && Packet->QueueType != NVME_IO_QUEUE) {
492 return EFI_INVALID_PARAMETER;
493 }
494
495 //
496 // 'Attributes' with neither EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL nor
497 // EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL set is an illegal
498 // configuration.
499 //
500 Attributes = This->Mode->Attributes;
501 if ((Attributes & (EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL |
502 EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL)) == 0) {
503 return EFI_INVALID_PARAMETER;
504 }
505
506 //
507 // Buffer alignment check for TransferBuffer & MetadataBuffer.
508 //
509 IoAlign = This->Mode->IoAlign;
510 if (IoAlign > 0 && (((UINTN) Packet->TransferBuffer & (IoAlign - 1)) != 0)) {
511 return EFI_INVALID_PARAMETER;
512 }
513
514 if (IoAlign > 0 && (((UINTN) Packet->MetadataBuffer & (IoAlign - 1)) != 0)) {
515 return EFI_INVALID_PARAMETER;
516 }
517
518 Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
519
520 //
521 // Check NamespaceId is valid or not.
522 //
523 if ((NamespaceId > Private->ControllerData->Nn) &&
524 (NamespaceId != (UINT32) -1)) {
525 return EFI_INVALID_PARAMETER;
526 }
527
528 //
529 // Check whether TransferLength exceeds the maximum data transfer size.
530 //
531 if (Private->ControllerData->Mdts != 0) {
532 MaxTransLen = (1 << (Private->ControllerData->Mdts)) *
533 (1 << (Private->Cap.Mpsmin + 12));
534 if (Packet->TransferLength > MaxTransLen) {
535 Packet->TransferLength = MaxTransLen;
536 return EFI_BAD_BUFFER_SIZE;
537 }
538 }
539
540 PciIo = Private->PciIo;
541 MapData = NULL;
542 MapMeta = NULL;
543 MapPrpList = NULL;
544 PrpListHost = NULL;
545 PrpListNo = 0;
546 Prp = NULL;
547 TimerEvent = NULL;
548 Status = EFI_SUCCESS;
549
550 if (Packet->QueueType == NVME_ADMIN_QUEUE) {
551 QueueId = 0;
552 } else {
553 if (Event == NULL) {
554 QueueId = 1;
555 } else {
556 QueueId = 2;
557
558 //
559 // Submission queue full check.
560 //
561 if ((Private->SqTdbl[QueueId].Sqt + 1) % (NVME_ASYNC_CSQ_SIZE + 1) ==
562 Private->AsyncSqHead) {
563 return EFI_NOT_READY;
564 }
565 }
566 }
567 Sq = Private->SqBuffer[QueueId] + Private->SqTdbl[QueueId].Sqt;
568 Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
569
570 if (Packet->NvmeCmd->Nsid != NamespaceId) {
571 return EFI_INVALID_PARAMETER;
572 }
573
574 ZeroMem (Sq, sizeof (NVME_SQ));
575 Sq->Opc = (UINT8)Packet->NvmeCmd->Cdw0.Opcode;
576 Sq->Fuse = (UINT8)Packet->NvmeCmd->Cdw0.FusedOperation;
577 Sq->Cid = Private->Cid[QueueId]++;
578 Sq->Nsid = Packet->NvmeCmd->Nsid;
579
580 //
581 // Currently we only support PRP for data transfer, SGL is NOT supported.
582 //
583 ASSERT (Sq->Psdt == 0);
584 if (Sq->Psdt != 0) {
585 DEBUG ((EFI_D_ERROR, "NvmExpressPassThru: doesn't support SGL mechanism\n"));
586 return EFI_UNSUPPORTED;
587 }
588
589 Sq->Prp[0] = (UINT64)(UINTN)Packet->TransferBuffer;
590 //
591 // If the NVMe cmd has data in or out, then mapping the user buffer to the PCI controller specific addresses.
592 // Note here we don't handle data buffer for CreateIOSubmitionQueue and CreateIOCompletionQueue cmds because
593 // these two cmds are special which requires their data buffer must support simultaneous access by both the
594 // processor and a PCI Bus Master. It's caller's responsbility to ensure this.
595 //
596 if (((Sq->Opc & (BIT0 | BIT1)) != 0) && (Sq->Opc != NVME_ADMIN_CRIOCQ_CMD) && (Sq->Opc != NVME_ADMIN_CRIOSQ_CMD)) {
597 if ((Packet->TransferLength == 0) || (Packet->TransferBuffer == NULL)) {
598 return EFI_INVALID_PARAMETER;
599 }
600
601 if ((Sq->Opc & BIT0) != 0) {
602 Flag = EfiPciIoOperationBusMasterRead;
603 } else {
604 Flag = EfiPciIoOperationBusMasterWrite;
605 }
606
607 MapLength = Packet->TransferLength;
608 Status = PciIo->Map (
609 PciIo,
610 Flag,
611 Packet->TransferBuffer,
612 &MapLength,
613 &PhyAddr,
614 &MapData
615 );
616 if (EFI_ERROR (Status) || (Packet->TransferLength != MapLength)) {
617 return EFI_OUT_OF_RESOURCES;
618 }
619
620 Sq->Prp[0] = PhyAddr;
621 Sq->Prp[1] = 0;
622
623 if((Packet->MetadataLength != 0) && (Packet->MetadataBuffer != NULL)) {
624 MapLength = Packet->MetadataLength;
625 Status = PciIo->Map (
626 PciIo,
627 Flag,
628 Packet->MetadataBuffer,
629 &MapLength,
630 &PhyAddr,
631 &MapMeta
632 );
633 if (EFI_ERROR (Status) || (Packet->MetadataLength != MapLength)) {
634 PciIo->Unmap (
635 PciIo,
636 MapData
637 );
638
639 return EFI_OUT_OF_RESOURCES;
640 }
641 Sq->Mptr = PhyAddr;
642 }
643 }
644 //
645 // If the buffer size spans more than two memory pages (page size as defined in CC.Mps),
646 // then build a PRP list in the second PRP submission queue entry.
647 //
648 Offset = ((UINT16)Sq->Prp[0]) & (EFI_PAGE_SIZE - 1);
649 Bytes = Packet->TransferLength;
650
651 if ((Offset + Bytes) > (EFI_PAGE_SIZE * 2)) {
652 //
653 // Create PrpList for remaining data buffer.
654 //
655 PhyAddr = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
656 Prp = NvmeCreatePrpList (PciIo, PhyAddr, EFI_SIZE_TO_PAGES(Offset + Bytes) - 1, &PrpListHost, &PrpListNo, &MapPrpList);
657 if (Prp == NULL) {
658 goto EXIT;
659 }
660
661 Sq->Prp[1] = (UINT64)(UINTN)Prp;
662 } else if ((Offset + Bytes) > EFI_PAGE_SIZE) {
663 Sq->Prp[1] = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
664 }
665
666 if(Packet->NvmeCmd->Flags & CDW2_VALID) {
667 Sq->Rsvd2 = (UINT64)Packet->NvmeCmd->Cdw2;
668 }
669 if(Packet->NvmeCmd->Flags & CDW3_VALID) {
670 Sq->Rsvd2 |= LShiftU64 ((UINT64)Packet->NvmeCmd->Cdw3, 32);
671 }
672 if(Packet->NvmeCmd->Flags & CDW10_VALID) {
673 Sq->Payload.Raw.Cdw10 = Packet->NvmeCmd->Cdw10;
674 }
675 if(Packet->NvmeCmd->Flags & CDW11_VALID) {
676 Sq->Payload.Raw.Cdw11 = Packet->NvmeCmd->Cdw11;
677 }
678 if(Packet->NvmeCmd->Flags & CDW12_VALID) {
679 Sq->Payload.Raw.Cdw12 = Packet->NvmeCmd->Cdw12;
680 }
681 if(Packet->NvmeCmd->Flags & CDW13_VALID) {
682 Sq->Payload.Raw.Cdw13 = Packet->NvmeCmd->Cdw13;
683 }
684 if(Packet->NvmeCmd->Flags & CDW14_VALID) {
685 Sq->Payload.Raw.Cdw14 = Packet->NvmeCmd->Cdw14;
686 }
687 if(Packet->NvmeCmd->Flags & CDW15_VALID) {
688 Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
689 }
690
691 //
692 // Ring the submission queue doorbell.
693 //
694 if ((Event != NULL) && (QueueId != 0)) {
695 Private->SqTdbl[QueueId].Sqt =
696 (Private->SqTdbl[QueueId].Sqt + 1) % (NVME_ASYNC_CSQ_SIZE + 1);
697 } else {
698 Private->SqTdbl[QueueId].Sqt ^= 1;
699 }
700 Data = ReadUnaligned32 ((UINT32*)&Private->SqTdbl[QueueId]);
701 Status = PciIo->Mem.Write (
702 PciIo,
703 EfiPciIoWidthUint32,
704 NVME_BAR,
705 NVME_SQTDBL_OFFSET(QueueId, Private->Cap.Dstrd),
706 1,
707 &Data
708 );
709
710 if (EFI_ERROR (Status)) {
711 goto EXIT;
712 }
713
714 //
715 // For non-blocking requests, return directly if the command is placed
716 // in the submission queue.
717 //
718 if ((Event != NULL) && (QueueId != 0)) {
719 AsyncRequest = AllocateZeroPool (sizeof (NVME_PASS_THRU_ASYNC_REQ));
720 if (AsyncRequest == NULL) {
721 Status = EFI_DEVICE_ERROR;
722 goto EXIT;
723 }
724
725 AsyncRequest->Signature = NVME_PASS_THRU_ASYNC_REQ_SIG;
726 AsyncRequest->Packet = Packet;
727 AsyncRequest->CommandId = Sq->Cid;
728 AsyncRequest->CallerEvent = Event;
729 AsyncRequest->MapData = MapData;
730 AsyncRequest->MapMeta = MapMeta;
731 AsyncRequest->MapPrpList = MapPrpList;
732 AsyncRequest->PrpListNo = PrpListNo;
733 AsyncRequest->PrpListHost = PrpListHost;
734
735 OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
736 InsertTailList (&Private->AsyncPassThruQueue, &AsyncRequest->Link);
737 gBS->RestoreTPL (OldTpl);
738
739 return EFI_SUCCESS;
740 }
741
742 Status = gBS->CreateEvent (
743 EVT_TIMER,
744 TPL_CALLBACK,
745 NULL,
746 NULL,
747 &TimerEvent
748 );
749 if (EFI_ERROR (Status)) {
750 goto EXIT;
751 }
752
753 Status = gBS->SetTimer(TimerEvent, TimerRelative, Packet->CommandTimeout);
754
755 if (EFI_ERROR(Status)) {
756 goto EXIT;
757 }
758
759 //
760 // Wait for completion queue to get filled in.
761 //
762 Status = EFI_TIMEOUT;
763 while (EFI_ERROR (gBS->CheckEvent (TimerEvent))) {
764 if (Cq->Pt != Private->Pt[QueueId]) {
765 Status = EFI_SUCCESS;
766 break;
767 }
768 }
769
770 //
771 // Check the NVMe cmd execution result
772 //
773 if (Status != EFI_TIMEOUT) {
774 if ((Cq->Sct == 0) && (Cq->Sc == 0)) {
775 Status = EFI_SUCCESS;
776 } else {
777 Status = EFI_DEVICE_ERROR;
778 //
779 // Copy the Respose Queue entry for this command to the callers response buffer
780 //
781 CopyMem(Packet->NvmeCompletion, Cq, sizeof(EFI_NVM_EXPRESS_COMPLETION));
782
783 //
784 // Dump every completion entry status for debugging.
785 //
786 DEBUG_CODE_BEGIN();
787 NvmeDumpStatus(Cq);
788 DEBUG_CODE_END();
789 }
790 } else {
791 //
792 // Timeout occurs for an NVMe command. Reset the controller to abort the
793 // outstanding commands.
794 //
795 DEBUG ((DEBUG_ERROR, "NvmExpressPassThru: Timeout occurs for an NVMe command.\n"));
796
797 //
798 // Disable the timer to trigger the process of async transfers temporarily.
799 //
800 Status = gBS->SetTimer (Private->TimerEvent, TimerCancel, 0);
801 if (EFI_ERROR (Status)) {
802 goto EXIT;
803 }
804
805 //
806 // Reset the NVMe controller.
807 //
808 Status = NvmeControllerInit (Private);
809 if (!EFI_ERROR (Status)) {
810 Status = AbortAsyncPassThruTasks (Private);
811 if (!EFI_ERROR (Status)) {
812 //
813 // Re-enable the timer to trigger the process of async transfers.
814 //
815 Status = gBS->SetTimer (Private->TimerEvent, TimerPeriodic, NVME_HC_ASYNC_TIMER);
816 if (!EFI_ERROR (Status)) {
817 //
818 // Return EFI_TIMEOUT to indicate a timeout occurs for NVMe PassThru command.
819 //
820 Status = EFI_TIMEOUT;
821 }
822 }
823 } else {
824 Status = EFI_DEVICE_ERROR;
825 }
826
827 goto EXIT;
828 }
829
830 if ((Private->CqHdbl[QueueId].Cqh ^= 1) == 0) {
831 Private->Pt[QueueId] ^= 1;
832 }
833
834 Data = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueId]);
835 PreviousStatus = Status;
836 Status = PciIo->Mem.Write (
837 PciIo,
838 EfiPciIoWidthUint32,
839 NVME_BAR,
840 NVME_CQHDBL_OFFSET(QueueId, Private->Cap.Dstrd),
841 1,
842 &Data
843 );
844 // The return status of PciIo->Mem.Write should not override
845 // previous status if previous status contains error.
846 Status = EFI_ERROR (PreviousStatus) ? PreviousStatus : Status;
847
848 //
849 // For now, the code does not support the non-blocking feature for admin queue.
850 // If Event is not NULL for admin queue, signal the caller's event here.
851 //
852 if (Event != NULL) {
853 ASSERT (QueueId == 0);
854 gBS->SignalEvent (Event);
855 }
856
857 EXIT:
858 if (MapData != NULL) {
859 PciIo->Unmap (
860 PciIo,
861 MapData
862 );
863 }
864
865 if (MapMeta != NULL) {
866 PciIo->Unmap (
867 PciIo,
868 MapMeta
869 );
870 }
871
872 if (MapPrpList != NULL) {
873 PciIo->Unmap (
874 PciIo,
875 MapPrpList
876 );
877 }
878
879 if (Prp != NULL) {
880 PciIo->FreeBuffer (PciIo, PrpListNo, PrpListHost);
881 }
882
883 if (TimerEvent != NULL) {
884 gBS->CloseEvent (TimerEvent);
885 }
886 return Status;
887 }
888
889 /**
890 Used to retrieve the next namespace ID for this NVM Express controller.
891
892 The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNextNamespace() function retrieves the next valid
893 namespace ID on this NVM Express controller.
894
895 If on input the value pointed to by NamespaceId is 0xFFFFFFFF, then the first valid namespace
896 ID defined on the NVM Express controller is returned in the location pointed to by NamespaceId
897 and a status of EFI_SUCCESS is returned.
898
899 If on input the value pointed to by NamespaceId is an invalid namespace ID other than 0xFFFFFFFF,
900 then EFI_INVALID_PARAMETER is returned.
901
902 If on input the value pointed to by NamespaceId is a valid namespace ID, then the next valid
903 namespace ID on the NVM Express controller is returned in the location pointed to by NamespaceId,
904 and EFI_SUCCESS is returned.
905
906 If the value pointed to by NamespaceId is the namespace ID of the last namespace on the NVM
907 Express controller, then EFI_NOT_FOUND is returned.
908
909 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
910 @param[in,out] NamespaceId On input, a pointer to a legal NamespaceId for an NVM Express
911 namespace present on the NVM Express controller. On output, a
912 pointer to the next NamespaceId of an NVM Express namespace on
913 an NVM Express controller. An input value of 0xFFFFFFFF retrieves
914 the first NamespaceId for an NVM Express namespace present on an
915 NVM Express controller.
916
917 @retval EFI_SUCCESS The Namespace ID of the next Namespace was returned.
918 @retval EFI_NOT_FOUND There are no more namespaces defined on this controller.
919 @retval EFI_INVALID_PARAMETER NamespaceId is an invalid value other than 0xFFFFFFFF.
920
921 **/
922 EFI_STATUS
923 EFIAPI
924 NvmExpressGetNextNamespace (
925 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *This,
926 IN OUT UINT32 *NamespaceId
927 )
928 {
929 NVME_CONTROLLER_PRIVATE_DATA *Private;
930 NVME_ADMIN_NAMESPACE_DATA *NamespaceData;
931 UINT32 NextNamespaceId;
932 EFI_STATUS Status;
933
934 if ((This == NULL) || (NamespaceId == NULL)) {
935 return EFI_INVALID_PARAMETER;
936 }
937
938 NamespaceData = NULL;
939 Status = EFI_NOT_FOUND;
940
941 Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
942 //
943 // If the NamespaceId input value is 0xFFFFFFFF, then get the first valid namespace ID
944 //
945 if (*NamespaceId == 0xFFFFFFFF) {
946 //
947 // Start with the first namespace ID
948 //
949 NextNamespaceId = 1;
950 //
951 // Allocate buffer for Identify Namespace data.
952 //
953 NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
954
955 if (NamespaceData == NULL) {
956 return EFI_NOT_FOUND;
957 }
958
959 Status = NvmeIdentifyNamespace (Private, NextNamespaceId, NamespaceData);
960 if (EFI_ERROR(Status)) {
961 goto Done;
962 }
963
964 *NamespaceId = NextNamespaceId;
965 } else {
966 if (*NamespaceId > Private->ControllerData->Nn) {
967 return EFI_INVALID_PARAMETER;
968 }
969
970 NextNamespaceId = *NamespaceId + 1;
971 if (NextNamespaceId > Private->ControllerData->Nn) {
972 return EFI_NOT_FOUND;
973 }
974
975 //
976 // Allocate buffer for Identify Namespace data.
977 //
978 NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
979 if (NamespaceData == NULL) {
980 return EFI_NOT_FOUND;
981 }
982
983 Status = NvmeIdentifyNamespace (Private, NextNamespaceId, NamespaceData);
984 if (EFI_ERROR(Status)) {
985 goto Done;
986 }
987
988 *NamespaceId = NextNamespaceId;
989 }
990
991 Done:
992 if (NamespaceData != NULL) {
993 FreePool(NamespaceData);
994 }
995
996 return Status;
997 }
998
999 /**
1000 Used to translate a device path node to a namespace ID.
1001
1002 The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNamespace() function determines the namespace ID associated with the
1003 namespace described by DevicePath.
1004
1005 If DevicePath is a device path node type that the NVM Express Pass Thru driver supports, then the NVM Express
1006 Pass Thru driver will attempt to translate the contents DevicePath into a namespace ID.
1007
1008 If this translation is successful, then that namespace ID is returned in NamespaceId, and EFI_SUCCESS is returned
1009
1010 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
1011 @param[in] DevicePath A pointer to the device path node that describes an NVM Express namespace on
1012 the NVM Express controller.
1013 @param[out] NamespaceId The NVM Express namespace ID contained in the device path node.
1014
1015 @retval EFI_SUCCESS DevicePath was successfully translated to NamespaceId.
1016 @retval EFI_INVALID_PARAMETER If DevicePath or NamespaceId are NULL, then EFI_INVALID_PARAMETER is returned.
1017 @retval EFI_UNSUPPORTED If DevicePath is not a device path node type that the NVM Express Pass Thru driver
1018 supports, then EFI_UNSUPPORTED is returned.
1019 @retval EFI_NOT_FOUND If DevicePath is a device path node type that the NVM Express Pass Thru driver
1020 supports, but there is not a valid translation from DevicePath to a namespace ID,
1021 then EFI_NOT_FOUND is returned.
1022 **/
1023 EFI_STATUS
1024 EFIAPI
1025 NvmExpressGetNamespace (
1026 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *This,
1027 IN EFI_DEVICE_PATH_PROTOCOL *DevicePath,
1028 OUT UINT32 *NamespaceId
1029 )
1030 {
1031 NVME_NAMESPACE_DEVICE_PATH *Node;
1032 NVME_CONTROLLER_PRIVATE_DATA *Private;
1033
1034 if ((This == NULL) || (DevicePath == NULL) || (NamespaceId == NULL)) {
1035 return EFI_INVALID_PARAMETER;
1036 }
1037
1038 if (DevicePath->Type != MESSAGING_DEVICE_PATH) {
1039 return EFI_UNSUPPORTED;
1040 }
1041
1042 Node = (NVME_NAMESPACE_DEVICE_PATH *)DevicePath;
1043 Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
1044
1045 if (DevicePath->SubType == MSG_NVME_NAMESPACE_DP) {
1046 if (DevicePathNodeLength(DevicePath) != sizeof(NVME_NAMESPACE_DEVICE_PATH)) {
1047 return EFI_NOT_FOUND;
1048 }
1049
1050 //
1051 // Check NamespaceId in the device path node is valid or not.
1052 //
1053 if ((Node->NamespaceId == 0) ||
1054 (Node->NamespaceId > Private->ControllerData->Nn)) {
1055 return EFI_NOT_FOUND;
1056 }
1057
1058 *NamespaceId = Node->NamespaceId;
1059
1060 return EFI_SUCCESS;
1061 } else {
1062 return EFI_UNSUPPORTED;
1063 }
1064 }
1065
1066 /**
1067 Used to allocate and build a device path node for an NVM Express namespace on an NVM Express controller.
1068
1069 The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.BuildDevicePath() function allocates and builds a single device
1070 path node for the NVM Express namespace specified by NamespaceId.
1071
1072 If the NamespaceId is not valid, then EFI_NOT_FOUND is returned.
1073
1074 If DevicePath is NULL, then EFI_INVALID_PARAMETER is returned.
1075
1076 If there are not enough resources to allocate the device path node, then EFI_OUT_OF_RESOURCES is returned.
1077
1078 Otherwise, DevicePath is allocated with the boot service AllocatePool(), the contents of DevicePath are
1079 initialized to describe the NVM Express namespace specified by NamespaceId, and EFI_SUCCESS is returned.
1080
1081 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
1082 @param[in] NamespaceId The NVM Express namespace ID for which a device path node is to be
1083 allocated and built. Caller must set the NamespaceId to zero if the
1084 device path node will contain a valid UUID.
1085 @param[in,out] DevicePath A pointer to a single device path node that describes the NVM Express
1086 namespace specified by NamespaceId. This function is responsible for
1087 allocating the buffer DevicePath with the boot service AllocatePool().
1088 It is the caller's responsibility to free DevicePath when the caller
1089 is finished with DevicePath.
1090 @retval EFI_SUCCESS The device path node that describes the NVM Express namespace specified
1091 by NamespaceId was allocated and returned in DevicePath.
1092 @retval EFI_NOT_FOUND The NamespaceId is not valid.
1093 @retval EFI_INVALID_PARAMETER DevicePath is NULL.
1094 @retval EFI_OUT_OF_RESOURCES There are not enough resources to allocate the DevicePath node.
1095
1096 **/
1097 EFI_STATUS
1098 EFIAPI
1099 NvmExpressBuildDevicePath (
1100 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *This,
1101 IN UINT32 NamespaceId,
1102 IN OUT EFI_DEVICE_PATH_PROTOCOL **DevicePath
1103 )
1104 {
1105 NVME_NAMESPACE_DEVICE_PATH *Node;
1106 NVME_CONTROLLER_PRIVATE_DATA *Private;
1107 EFI_STATUS Status;
1108 NVME_ADMIN_NAMESPACE_DATA *NamespaceData;
1109
1110 //
1111 // Validate parameters
1112 //
1113 if ((This == NULL) || (DevicePath == NULL)) {
1114 return EFI_INVALID_PARAMETER;
1115 }
1116
1117 Status = EFI_SUCCESS;
1118 Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
1119
1120 //
1121 // Check NamespaceId is valid or not.
1122 //
1123 if ((NamespaceId == 0) ||
1124 (NamespaceId > Private->ControllerData->Nn)) {
1125 return EFI_NOT_FOUND;
1126 }
1127
1128 Node = (NVME_NAMESPACE_DEVICE_PATH *)AllocateZeroPool (sizeof (NVME_NAMESPACE_DEVICE_PATH));
1129 if (Node == NULL) {
1130 return EFI_OUT_OF_RESOURCES;
1131 }
1132
1133 Node->Header.Type = MESSAGING_DEVICE_PATH;
1134 Node->Header.SubType = MSG_NVME_NAMESPACE_DP;
1135 SetDevicePathNodeLength (&Node->Header, sizeof (NVME_NAMESPACE_DEVICE_PATH));
1136 Node->NamespaceId = NamespaceId;
1137
1138 //
1139 // Allocate a buffer for Identify Namespace data.
1140 //
1141 NamespaceData = NULL;
1142 NamespaceData = AllocateZeroPool(sizeof (NVME_ADMIN_NAMESPACE_DATA));
1143 if(NamespaceData == NULL) {
1144 Status = EFI_OUT_OF_RESOURCES;
1145 goto Exit;
1146 }
1147
1148 //
1149 // Get UUID from specified Identify Namespace data.
1150 //
1151 Status = NvmeIdentifyNamespace (
1152 Private,
1153 NamespaceId,
1154 (VOID *)NamespaceData
1155 );
1156
1157 if (EFI_ERROR(Status)) {
1158 goto Exit;
1159 }
1160
1161 Node->NamespaceUuid = NamespaceData->Eui64;
1162
1163 *DevicePath = (EFI_DEVICE_PATH_PROTOCOL *)Node;
1164
1165 Exit:
1166 if(NamespaceData != NULL) {
1167 FreePool (NamespaceData);
1168 }
1169
1170 if (EFI_ERROR (Status)) {
1171 FreePool (Node);
1172 }
1173
1174 return Status;
1175 }
1176