]> git.proxmox.com Git - mirror_edk2.git/blob - MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c
MdeModulePkg: Replace BSD License with BSD+Patent License
[mirror_edk2.git] / MdeModulePkg / Bus / Pci / NvmExpressDxe / NvmExpressPassthru.c
1 /** @file
2 NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
3 NVM Express specification.
4
5 (C) Copyright 2014 Hewlett-Packard Development Company, L.P.<BR>
6 Copyright (c) 2013 - 2018, Intel Corporation. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
8
9 **/
10
11 #include "NvmExpress.h"
12
13 /**
14 Dump the execution status from a given completion queue entry.
15
16 @param[in] Cq A pointer to the NVME_CQ item.
17
18 **/
19 VOID
20 NvmeDumpStatus (
21 IN NVME_CQ *Cq
22 )
23 {
24 DEBUG ((EFI_D_VERBOSE, "Dump NVMe Completion Entry Status from [0x%x]:\n", Cq));
25
26 DEBUG ((EFI_D_VERBOSE, " SQ Identifier : [0x%x], Phase Tag : [%d], Cmd Identifier : [0x%x]\n", Cq->Sqid, Cq->Pt, Cq->Cid));
27
28 DEBUG ((EFI_D_VERBOSE, " NVMe Cmd Execution Result - "));
29
30 switch (Cq->Sct) {
31 case 0x0:
32 switch (Cq->Sc) {
33 case 0x0:
34 DEBUG ((EFI_D_VERBOSE, "Successful Completion\n"));
35 break;
36 case 0x1:
37 DEBUG ((EFI_D_VERBOSE, "Invalid Command Opcode\n"));
38 break;
39 case 0x2:
40 DEBUG ((EFI_D_VERBOSE, "Invalid Field in Command\n"));
41 break;
42 case 0x3:
43 DEBUG ((EFI_D_VERBOSE, "Command ID Conflict\n"));
44 break;
45 case 0x4:
46 DEBUG ((EFI_D_VERBOSE, "Data Transfer Error\n"));
47 break;
48 case 0x5:
49 DEBUG ((EFI_D_VERBOSE, "Commands Aborted due to Power Loss Notification\n"));
50 break;
51 case 0x6:
52 DEBUG ((EFI_D_VERBOSE, "Internal Device Error\n"));
53 break;
54 case 0x7:
55 DEBUG ((EFI_D_VERBOSE, "Command Abort Requested\n"));
56 break;
57 case 0x8:
58 DEBUG ((EFI_D_VERBOSE, "Command Aborted due to SQ Deletion\n"));
59 break;
60 case 0x9:
61 DEBUG ((EFI_D_VERBOSE, "Command Aborted due to Failed Fused Command\n"));
62 break;
63 case 0xA:
64 DEBUG ((EFI_D_VERBOSE, "Command Aborted due to Missing Fused Command\n"));
65 break;
66 case 0xB:
67 DEBUG ((EFI_D_VERBOSE, "Invalid Namespace or Format\n"));
68 break;
69 case 0xC:
70 DEBUG ((EFI_D_VERBOSE, "Command Sequence Error\n"));
71 break;
72 case 0xD:
73 DEBUG ((EFI_D_VERBOSE, "Invalid SGL Last Segment Descriptor\n"));
74 break;
75 case 0xE:
76 DEBUG ((EFI_D_VERBOSE, "Invalid Number of SGL Descriptors\n"));
77 break;
78 case 0xF:
79 DEBUG ((EFI_D_VERBOSE, "Data SGL Length Invalid\n"));
80 break;
81 case 0x10:
82 DEBUG ((EFI_D_VERBOSE, "Metadata SGL Length Invalid\n"));
83 break;
84 case 0x11:
85 DEBUG ((EFI_D_VERBOSE, "SGL Descriptor Type Invalid\n"));
86 break;
87 case 0x80:
88 DEBUG ((EFI_D_VERBOSE, "LBA Out of Range\n"));
89 break;
90 case 0x81:
91 DEBUG ((EFI_D_VERBOSE, "Capacity Exceeded\n"));
92 break;
93 case 0x82:
94 DEBUG ((EFI_D_VERBOSE, "Namespace Not Ready\n"));
95 break;
96 case 0x83:
97 DEBUG ((EFI_D_VERBOSE, "Reservation Conflict\n"));
98 break;
99 }
100 break;
101
102 case 0x1:
103 switch (Cq->Sc) {
104 case 0x0:
105 DEBUG ((EFI_D_VERBOSE, "Completion Queue Invalid\n"));
106 break;
107 case 0x1:
108 DEBUG ((EFI_D_VERBOSE, "Invalid Queue Identifier\n"));
109 break;
110 case 0x2:
111 DEBUG ((EFI_D_VERBOSE, "Maximum Queue Size Exceeded\n"));
112 break;
113 case 0x3:
114 DEBUG ((EFI_D_VERBOSE, "Abort Command Limit Exceeded\n"));
115 break;
116 case 0x5:
117 DEBUG ((EFI_D_VERBOSE, "Asynchronous Event Request Limit Exceeded\n"));
118 break;
119 case 0x6:
120 DEBUG ((EFI_D_VERBOSE, "Invalid Firmware Slot\n"));
121 break;
122 case 0x7:
123 DEBUG ((EFI_D_VERBOSE, "Invalid Firmware Image\n"));
124 break;
125 case 0x8:
126 DEBUG ((EFI_D_VERBOSE, "Invalid Interrupt Vector\n"));
127 break;
128 case 0x9:
129 DEBUG ((EFI_D_VERBOSE, "Invalid Log Page\n"));
130 break;
131 case 0xA:
132 DEBUG ((EFI_D_VERBOSE, "Invalid Format\n"));
133 break;
134 case 0xB:
135 DEBUG ((EFI_D_VERBOSE, "Firmware Application Requires Conventional Reset\n"));
136 break;
137 case 0xC:
138 DEBUG ((EFI_D_VERBOSE, "Invalid Queue Deletion\n"));
139 break;
140 case 0xD:
141 DEBUG ((EFI_D_VERBOSE, "Feature Identifier Not Saveable\n"));
142 break;
143 case 0xE:
144 DEBUG ((EFI_D_VERBOSE, "Feature Not Changeable\n"));
145 break;
146 case 0xF:
147 DEBUG ((EFI_D_VERBOSE, "Feature Not Namespace Specific\n"));
148 break;
149 case 0x10:
150 DEBUG ((EFI_D_VERBOSE, "Firmware Application Requires NVM Subsystem Reset\n"));
151 break;
152 case 0x80:
153 DEBUG ((EFI_D_VERBOSE, "Conflicting Attributes\n"));
154 break;
155 case 0x81:
156 DEBUG ((EFI_D_VERBOSE, "Invalid Protection Information\n"));
157 break;
158 case 0x82:
159 DEBUG ((EFI_D_VERBOSE, "Attempted Write to Read Only Range\n"));
160 break;
161 }
162 break;
163
164 case 0x2:
165 switch (Cq->Sc) {
166 case 0x80:
167 DEBUG ((EFI_D_VERBOSE, "Write Fault\n"));
168 break;
169 case 0x81:
170 DEBUG ((EFI_D_VERBOSE, "Unrecovered Read Error\n"));
171 break;
172 case 0x82:
173 DEBUG ((EFI_D_VERBOSE, "End-to-end Guard Check Error\n"));
174 break;
175 case 0x83:
176 DEBUG ((EFI_D_VERBOSE, "End-to-end Application Tag Check Error\n"));
177 break;
178 case 0x84:
179 DEBUG ((EFI_D_VERBOSE, "End-to-end Reference Tag Check Error\n"));
180 break;
181 case 0x85:
182 DEBUG ((EFI_D_VERBOSE, "Compare Failure\n"));
183 break;
184 case 0x86:
185 DEBUG ((EFI_D_VERBOSE, "Access Denied\n"));
186 break;
187 }
188 break;
189
190 default:
191 break;
192 }
193 }
194
195 /**
196 Create PRP lists for data transfer which is larger than 2 memory pages.
197 Note here we calcuate the number of required PRP lists and allocate them at one time.
198
199 @param[in] PciIo A pointer to the EFI_PCI_IO_PROTOCOL instance.
200 @param[in] PhysicalAddr The physical base address of data buffer.
201 @param[in] Pages The number of pages to be transfered.
202 @param[out] PrpListHost The host base address of PRP lists.
203 @param[in,out] PrpListNo The number of PRP List.
204 @param[out] Mapping The mapping value returned from PciIo.Map().
205
206 @retval The pointer to the first PRP List of the PRP lists.
207
208 **/
209 VOID*
210 NvmeCreatePrpList (
211 IN EFI_PCI_IO_PROTOCOL *PciIo,
212 IN EFI_PHYSICAL_ADDRESS PhysicalAddr,
213 IN UINTN Pages,
214 OUT VOID **PrpListHost,
215 IN OUT UINTN *PrpListNo,
216 OUT VOID **Mapping
217 )
218 {
219 UINTN PrpEntryNo;
220 UINT64 PrpListBase;
221 UINTN PrpListIndex;
222 UINTN PrpEntryIndex;
223 UINT64 Remainder;
224 EFI_PHYSICAL_ADDRESS PrpListPhyAddr;
225 UINTN Bytes;
226 EFI_STATUS Status;
227
228 //
229 // The number of Prp Entry in a memory page.
230 //
231 PrpEntryNo = EFI_PAGE_SIZE / sizeof (UINT64);
232
233 //
234 // Calculate total PrpList number.
235 //
236 *PrpListNo = (UINTN)DivU64x64Remainder ((UINT64)Pages, (UINT64)PrpEntryNo - 1, &Remainder);
237 if (*PrpListNo == 0) {
238 *PrpListNo = 1;
239 } else if ((Remainder != 0) && (Remainder != 1)) {
240 *PrpListNo += 1;
241 } else if (Remainder == 1) {
242 Remainder = PrpEntryNo;
243 } else if (Remainder == 0) {
244 Remainder = PrpEntryNo - 1;
245 }
246
247 Status = PciIo->AllocateBuffer (
248 PciIo,
249 AllocateAnyPages,
250 EfiBootServicesData,
251 *PrpListNo,
252 PrpListHost,
253 0
254 );
255
256 if (EFI_ERROR (Status)) {
257 return NULL;
258 }
259
260 Bytes = EFI_PAGES_TO_SIZE (*PrpListNo);
261 Status = PciIo->Map (
262 PciIo,
263 EfiPciIoOperationBusMasterCommonBuffer,
264 *PrpListHost,
265 &Bytes,
266 &PrpListPhyAddr,
267 Mapping
268 );
269
270 if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (*PrpListNo))) {
271 DEBUG ((EFI_D_ERROR, "NvmeCreatePrpList: create PrpList failure!\n"));
272 goto EXIT;
273 }
274 //
275 // Fill all PRP lists except of last one.
276 //
277 ZeroMem (*PrpListHost, Bytes);
278 for (PrpListIndex = 0; PrpListIndex < *PrpListNo - 1; ++PrpListIndex) {
279 PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
280
281 for (PrpEntryIndex = 0; PrpEntryIndex < PrpEntryNo; ++PrpEntryIndex) {
282 if (PrpEntryIndex != PrpEntryNo - 1) {
283 //
284 // Fill all PRP entries except of last one.
285 //
286 *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PhysicalAddr;
287 PhysicalAddr += EFI_PAGE_SIZE;
288 } else {
289 //
290 // Fill last PRP entries with next PRP List pointer.
291 //
292 *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PrpListPhyAddr + (PrpListIndex + 1) * EFI_PAGE_SIZE;
293 }
294 }
295 }
296 //
297 // Fill last PRP list.
298 //
299 PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
300 for (PrpEntryIndex = 0; PrpEntryIndex < Remainder; ++PrpEntryIndex) {
301 *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PhysicalAddr;
302 PhysicalAddr += EFI_PAGE_SIZE;
303 }
304
305 return (VOID*)(UINTN)PrpListPhyAddr;
306
307 EXIT:
308 PciIo->FreeBuffer (PciIo, *PrpListNo, *PrpListHost);
309 return NULL;
310 }
311
312
313 /**
314 Aborts the asynchronous PassThru requests.
315
316 @param[in] Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA
317 data structure.
318
319 @retval EFI_SUCCESS The asynchronous PassThru requests have been aborted.
320 @return EFI_DEVICE_ERROR Fail to abort all the asynchronous PassThru requests.
321
322 **/
323 EFI_STATUS
324 AbortAsyncPassThruTasks (
325 IN NVME_CONTROLLER_PRIVATE_DATA *Private
326 )
327 {
328 EFI_PCI_IO_PROTOCOL *PciIo;
329 LIST_ENTRY *Link;
330 LIST_ENTRY *NextLink;
331 NVME_BLKIO2_SUBTASK *Subtask;
332 NVME_BLKIO2_REQUEST *BlkIo2Request;
333 NVME_PASS_THRU_ASYNC_REQ *AsyncRequest;
334 EFI_BLOCK_IO2_TOKEN *Token;
335 EFI_TPL OldTpl;
336 EFI_STATUS Status;
337
338 PciIo = Private->PciIo;
339 OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
340
341 //
342 // Cancel the unsubmitted subtasks.
343 //
344 for (Link = GetFirstNode (&Private->UnsubmittedSubtasks);
345 !IsNull (&Private->UnsubmittedSubtasks, Link);
346 Link = NextLink) {
347 NextLink = GetNextNode (&Private->UnsubmittedSubtasks, Link);
348 Subtask = NVME_BLKIO2_SUBTASK_FROM_LINK (Link);
349 BlkIo2Request = Subtask->BlockIo2Request;
350 Token = BlkIo2Request->Token;
351
352 BlkIo2Request->UnsubmittedSubtaskNum--;
353 if (Subtask->IsLast) {
354 BlkIo2Request->LastSubtaskSubmitted = TRUE;
355 }
356 Token->TransactionStatus = EFI_ABORTED;
357
358 RemoveEntryList (Link);
359 InsertTailList (&BlkIo2Request->SubtasksQueue, Link);
360 gBS->SignalEvent (Subtask->Event);
361 }
362
363 //
364 // Cleanup the resources for the asynchronous PassThru requests.
365 //
366 for (Link = GetFirstNode (&Private->AsyncPassThruQueue);
367 !IsNull (&Private->AsyncPassThruQueue, Link);
368 Link = NextLink) {
369 NextLink = GetNextNode (&Private->AsyncPassThruQueue, Link);
370 AsyncRequest = NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link);
371
372 if (AsyncRequest->MapData != NULL) {
373 PciIo->Unmap (PciIo, AsyncRequest->MapData);
374 }
375 if (AsyncRequest->MapMeta != NULL) {
376 PciIo->Unmap (PciIo, AsyncRequest->MapMeta);
377 }
378 if (AsyncRequest->MapPrpList != NULL) {
379 PciIo->Unmap (PciIo, AsyncRequest->MapPrpList);
380 }
381 if (AsyncRequest->PrpListHost != NULL) {
382 PciIo->FreeBuffer (
383 PciIo,
384 AsyncRequest->PrpListNo,
385 AsyncRequest->PrpListHost
386 );
387 }
388
389 RemoveEntryList (Link);
390 gBS->SignalEvent (AsyncRequest->CallerEvent);
391 FreePool (AsyncRequest);
392 }
393
394 if (IsListEmpty (&Private->AsyncPassThruQueue) &&
395 IsListEmpty (&Private->UnsubmittedSubtasks)) {
396 Status = EFI_SUCCESS;
397 } else {
398 Status = EFI_DEVICE_ERROR;
399 }
400
401 gBS->RestoreTPL (OldTpl);
402
403 return Status;
404 }
405
406
407 /**
408 Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function supports
409 both blocking I/O and non-blocking I/O. The blocking I/O functionality is required, and the non-blocking
410 I/O functionality is optional.
411
412
413 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
414 @param[in] NamespaceId A 32 bit namespace ID as defined in the NVMe specification to which the NVM Express Command
415 Packet will be sent. A value of 0 denotes the NVM Express controller, a value of all 0xFF's
416 (all bytes are 0xFF) in the namespace ID specifies that the command packet should be sent to
417 all valid namespaces.
418 @param[in,out] Packet A pointer to the NVM Express Command Packet.
419 @param[in] Event If non-blocking I/O is not supported then Event is ignored, and blocking I/O is performed.
420 If Event is NULL, then blocking I/O is performed. If Event is not NULL and non-blocking I/O
421 is supported, then non-blocking I/O is performed, and Event will be signaled when the NVM
422 Express Command Packet completes.
423
424 @retval EFI_SUCCESS The NVM Express Command Packet was sent by the host. TransferLength bytes were transferred
425 to, or from DataBuffer.
426 @retval EFI_BAD_BUFFER_SIZE The NVM Express Command Packet was not executed. The number of bytes that could be transferred
427 is returned in TransferLength.
428 @retval EFI_NOT_READY The NVM Express Command Packet could not be sent because the controller is not ready. The caller
429 may retry again later.
430 @retval EFI_DEVICE_ERROR A device error occurred while attempting to send the NVM Express Command Packet.
431 @retval EFI_INVALID_PARAMETER NamespaceId or the contents of EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET are invalid. The NVM
432 Express Command Packet was not sent, so no additional status information is available.
433 @retval EFI_UNSUPPORTED The command described by the NVM Express Command Packet is not supported by the NVM Express
434 controller. The NVM Express Command Packet was not sent so no additional status information
435 is available.
436 @retval EFI_TIMEOUT A timeout occurred while waiting for the NVM Express Command Packet to execute.
437
438 **/
439 EFI_STATUS
440 EFIAPI
441 NvmExpressPassThru (
442 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *This,
443 IN UINT32 NamespaceId,
444 IN OUT EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *Packet,
445 IN EFI_EVENT Event OPTIONAL
446 )
447 {
448 NVME_CONTROLLER_PRIVATE_DATA *Private;
449 EFI_STATUS Status;
450 EFI_STATUS PreviousStatus;
451 EFI_PCI_IO_PROTOCOL *PciIo;
452 NVME_SQ *Sq;
453 NVME_CQ *Cq;
454 UINT16 QueueId;
455 UINT32 Bytes;
456 UINT16 Offset;
457 EFI_EVENT TimerEvent;
458 EFI_PCI_IO_PROTOCOL_OPERATION Flag;
459 EFI_PHYSICAL_ADDRESS PhyAddr;
460 VOID *MapData;
461 VOID *MapMeta;
462 VOID *MapPrpList;
463 UINTN MapLength;
464 UINT64 *Prp;
465 VOID *PrpListHost;
466 UINTN PrpListNo;
467 UINT32 Attributes;
468 UINT32 IoAlign;
469 UINT32 MaxTransLen;
470 UINT32 Data;
471 NVME_PASS_THRU_ASYNC_REQ *AsyncRequest;
472 EFI_TPL OldTpl;
473
474 //
475 // check the data fields in Packet parameter.
476 //
477 if ((This == NULL) || (Packet == NULL)) {
478 return EFI_INVALID_PARAMETER;
479 }
480
481 if ((Packet->NvmeCmd == NULL) || (Packet->NvmeCompletion == NULL)) {
482 return EFI_INVALID_PARAMETER;
483 }
484
485 if (Packet->QueueType != NVME_ADMIN_QUEUE && Packet->QueueType != NVME_IO_QUEUE) {
486 return EFI_INVALID_PARAMETER;
487 }
488
489 //
490 // 'Attributes' with neither EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL nor
491 // EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL set is an illegal
492 // configuration.
493 //
494 Attributes = This->Mode->Attributes;
495 if ((Attributes & (EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL |
496 EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL)) == 0) {
497 return EFI_INVALID_PARAMETER;
498 }
499
500 //
501 // Buffer alignment check for TransferBuffer & MetadataBuffer.
502 //
503 IoAlign = This->Mode->IoAlign;
504 if (IoAlign > 0 && (((UINTN) Packet->TransferBuffer & (IoAlign - 1)) != 0)) {
505 return EFI_INVALID_PARAMETER;
506 }
507
508 if (IoAlign > 0 && (((UINTN) Packet->MetadataBuffer & (IoAlign - 1)) != 0)) {
509 return EFI_INVALID_PARAMETER;
510 }
511
512 Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
513
514 //
515 // Check NamespaceId is valid or not.
516 //
517 if ((NamespaceId > Private->ControllerData->Nn) &&
518 (NamespaceId != (UINT32) -1)) {
519 return EFI_INVALID_PARAMETER;
520 }
521
522 //
523 // Check whether TransferLength exceeds the maximum data transfer size.
524 //
525 if (Private->ControllerData->Mdts != 0) {
526 MaxTransLen = (1 << (Private->ControllerData->Mdts)) *
527 (1 << (Private->Cap.Mpsmin + 12));
528 if (Packet->TransferLength > MaxTransLen) {
529 Packet->TransferLength = MaxTransLen;
530 return EFI_BAD_BUFFER_SIZE;
531 }
532 }
533
534 PciIo = Private->PciIo;
535 MapData = NULL;
536 MapMeta = NULL;
537 MapPrpList = NULL;
538 PrpListHost = NULL;
539 PrpListNo = 0;
540 Prp = NULL;
541 TimerEvent = NULL;
542 Status = EFI_SUCCESS;
543
544 if (Packet->QueueType == NVME_ADMIN_QUEUE) {
545 QueueId = 0;
546 } else {
547 if (Event == NULL) {
548 QueueId = 1;
549 } else {
550 QueueId = 2;
551
552 //
553 // Submission queue full check.
554 //
555 if ((Private->SqTdbl[QueueId].Sqt + 1) % (NVME_ASYNC_CSQ_SIZE + 1) ==
556 Private->AsyncSqHead) {
557 return EFI_NOT_READY;
558 }
559 }
560 }
561 Sq = Private->SqBuffer[QueueId] + Private->SqTdbl[QueueId].Sqt;
562 Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
563
564 if (Packet->NvmeCmd->Nsid != NamespaceId) {
565 return EFI_INVALID_PARAMETER;
566 }
567
568 ZeroMem (Sq, sizeof (NVME_SQ));
569 Sq->Opc = (UINT8)Packet->NvmeCmd->Cdw0.Opcode;
570 Sq->Fuse = (UINT8)Packet->NvmeCmd->Cdw0.FusedOperation;
571 Sq->Cid = Private->Cid[QueueId]++;
572 Sq->Nsid = Packet->NvmeCmd->Nsid;
573
574 //
575 // Currently we only support PRP for data transfer, SGL is NOT supported.
576 //
577 ASSERT (Sq->Psdt == 0);
578 if (Sq->Psdt != 0) {
579 DEBUG ((EFI_D_ERROR, "NvmExpressPassThru: doesn't support SGL mechanism\n"));
580 return EFI_UNSUPPORTED;
581 }
582
583 Sq->Prp[0] = (UINT64)(UINTN)Packet->TransferBuffer;
584 if ((Packet->QueueType == NVME_ADMIN_QUEUE) &&
585 ((Sq->Opc == NVME_ADMIN_CRIOCQ_CMD) || (Sq->Opc == NVME_ADMIN_CRIOSQ_CMD))) {
586 //
587 // Currently, we only use the IO Completion/Submission queues created internally
588 // by this driver during controller initialization. Any other IO queues created
589 // will not be consumed here. The value is little to accept external IO queue
590 // creation requests, so here we will return EFI_UNSUPPORTED for external IO
591 // queue creation request.
592 //
593 if (!Private->CreateIoQueue) {
594 DEBUG ((DEBUG_ERROR, "NvmExpressPassThru: Does not support external IO queues creation request.\n"));
595 return EFI_UNSUPPORTED;
596 }
597 } else if ((Sq->Opc & (BIT0 | BIT1)) != 0) {
598 //
599 // If the NVMe cmd has data in or out, then mapping the user buffer to the PCI controller specific addresses.
600 //
601 if (((Packet->TransferLength != 0) && (Packet->TransferBuffer == NULL)) ||
602 ((Packet->TransferLength == 0) && (Packet->TransferBuffer != NULL))) {
603 return EFI_INVALID_PARAMETER;
604 }
605
606 if ((Sq->Opc & BIT0) != 0) {
607 Flag = EfiPciIoOperationBusMasterRead;
608 } else {
609 Flag = EfiPciIoOperationBusMasterWrite;
610 }
611
612 if ((Packet->TransferLength != 0) && (Packet->TransferBuffer != NULL)) {
613 MapLength = Packet->TransferLength;
614 Status = PciIo->Map (
615 PciIo,
616 Flag,
617 Packet->TransferBuffer,
618 &MapLength,
619 &PhyAddr,
620 &MapData
621 );
622 if (EFI_ERROR (Status) || (Packet->TransferLength != MapLength)) {
623 return EFI_OUT_OF_RESOURCES;
624 }
625
626 Sq->Prp[0] = PhyAddr;
627 Sq->Prp[1] = 0;
628 }
629
630 if((Packet->MetadataLength != 0) && (Packet->MetadataBuffer != NULL)) {
631 MapLength = Packet->MetadataLength;
632 Status = PciIo->Map (
633 PciIo,
634 Flag,
635 Packet->MetadataBuffer,
636 &MapLength,
637 &PhyAddr,
638 &MapMeta
639 );
640 if (EFI_ERROR (Status) || (Packet->MetadataLength != MapLength)) {
641 PciIo->Unmap (
642 PciIo,
643 MapData
644 );
645
646 return EFI_OUT_OF_RESOURCES;
647 }
648 Sq->Mptr = PhyAddr;
649 }
650 }
651 //
652 // If the buffer size spans more than two memory pages (page size as defined in CC.Mps),
653 // then build a PRP list in the second PRP submission queue entry.
654 //
655 Offset = ((UINT16)Sq->Prp[0]) & (EFI_PAGE_SIZE - 1);
656 Bytes = Packet->TransferLength;
657
658 if ((Offset + Bytes) > (EFI_PAGE_SIZE * 2)) {
659 //
660 // Create PrpList for remaining data buffer.
661 //
662 PhyAddr = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
663 Prp = NvmeCreatePrpList (PciIo, PhyAddr, EFI_SIZE_TO_PAGES(Offset + Bytes) - 1, &PrpListHost, &PrpListNo, &MapPrpList);
664 if (Prp == NULL) {
665 Status = EFI_OUT_OF_RESOURCES;
666 goto EXIT;
667 }
668
669 Sq->Prp[1] = (UINT64)(UINTN)Prp;
670 } else if ((Offset + Bytes) > EFI_PAGE_SIZE) {
671 Sq->Prp[1] = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
672 }
673
674 if(Packet->NvmeCmd->Flags & CDW2_VALID) {
675 Sq->Rsvd2 = (UINT64)Packet->NvmeCmd->Cdw2;
676 }
677 if(Packet->NvmeCmd->Flags & CDW3_VALID) {
678 Sq->Rsvd2 |= LShiftU64 ((UINT64)Packet->NvmeCmd->Cdw3, 32);
679 }
680 if(Packet->NvmeCmd->Flags & CDW10_VALID) {
681 Sq->Payload.Raw.Cdw10 = Packet->NvmeCmd->Cdw10;
682 }
683 if(Packet->NvmeCmd->Flags & CDW11_VALID) {
684 Sq->Payload.Raw.Cdw11 = Packet->NvmeCmd->Cdw11;
685 }
686 if(Packet->NvmeCmd->Flags & CDW12_VALID) {
687 Sq->Payload.Raw.Cdw12 = Packet->NvmeCmd->Cdw12;
688 }
689 if(Packet->NvmeCmd->Flags & CDW13_VALID) {
690 Sq->Payload.Raw.Cdw13 = Packet->NvmeCmd->Cdw13;
691 }
692 if(Packet->NvmeCmd->Flags & CDW14_VALID) {
693 Sq->Payload.Raw.Cdw14 = Packet->NvmeCmd->Cdw14;
694 }
695 if(Packet->NvmeCmd->Flags & CDW15_VALID) {
696 Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
697 }
698
699 //
700 // Ring the submission queue doorbell.
701 //
702 if ((Event != NULL) && (QueueId != 0)) {
703 Private->SqTdbl[QueueId].Sqt =
704 (Private->SqTdbl[QueueId].Sqt + 1) % (NVME_ASYNC_CSQ_SIZE + 1);
705 } else {
706 Private->SqTdbl[QueueId].Sqt ^= 1;
707 }
708 Data = ReadUnaligned32 ((UINT32*)&Private->SqTdbl[QueueId]);
709 Status = PciIo->Mem.Write (
710 PciIo,
711 EfiPciIoWidthUint32,
712 NVME_BAR,
713 NVME_SQTDBL_OFFSET(QueueId, Private->Cap.Dstrd),
714 1,
715 &Data
716 );
717
718 if (EFI_ERROR (Status)) {
719 goto EXIT;
720 }
721
722 //
723 // For non-blocking requests, return directly if the command is placed
724 // in the submission queue.
725 //
726 if ((Event != NULL) && (QueueId != 0)) {
727 AsyncRequest = AllocateZeroPool (sizeof (NVME_PASS_THRU_ASYNC_REQ));
728 if (AsyncRequest == NULL) {
729 Status = EFI_DEVICE_ERROR;
730 goto EXIT;
731 }
732
733 AsyncRequest->Signature = NVME_PASS_THRU_ASYNC_REQ_SIG;
734 AsyncRequest->Packet = Packet;
735 AsyncRequest->CommandId = Sq->Cid;
736 AsyncRequest->CallerEvent = Event;
737 AsyncRequest->MapData = MapData;
738 AsyncRequest->MapMeta = MapMeta;
739 AsyncRequest->MapPrpList = MapPrpList;
740 AsyncRequest->PrpListNo = PrpListNo;
741 AsyncRequest->PrpListHost = PrpListHost;
742
743 OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
744 InsertTailList (&Private->AsyncPassThruQueue, &AsyncRequest->Link);
745 gBS->RestoreTPL (OldTpl);
746
747 return EFI_SUCCESS;
748 }
749
750 Status = gBS->CreateEvent (
751 EVT_TIMER,
752 TPL_CALLBACK,
753 NULL,
754 NULL,
755 &TimerEvent
756 );
757 if (EFI_ERROR (Status)) {
758 goto EXIT;
759 }
760
761 Status = gBS->SetTimer(TimerEvent, TimerRelative, Packet->CommandTimeout);
762
763 if (EFI_ERROR(Status)) {
764 goto EXIT;
765 }
766
767 //
768 // Wait for completion queue to get filled in.
769 //
770 Status = EFI_TIMEOUT;
771 while (EFI_ERROR (gBS->CheckEvent (TimerEvent))) {
772 if (Cq->Pt != Private->Pt[QueueId]) {
773 Status = EFI_SUCCESS;
774 break;
775 }
776 }
777
778 //
779 // Check the NVMe cmd execution result
780 //
781 if (Status != EFI_TIMEOUT) {
782 if ((Cq->Sct == 0) && (Cq->Sc == 0)) {
783 Status = EFI_SUCCESS;
784 } else {
785 Status = EFI_DEVICE_ERROR;
786 //
787 // Dump every completion entry status for debugging.
788 //
789 DEBUG_CODE_BEGIN();
790 NvmeDumpStatus(Cq);
791 DEBUG_CODE_END();
792 }
793 //
794 // Copy the Respose Queue entry for this command to the callers response buffer
795 //
796 CopyMem(Packet->NvmeCompletion, Cq, sizeof(EFI_NVM_EXPRESS_COMPLETION));
797 } else {
798 //
799 // Timeout occurs for an NVMe command. Reset the controller to abort the
800 // outstanding commands.
801 //
802 DEBUG ((DEBUG_ERROR, "NvmExpressPassThru: Timeout occurs for an NVMe command.\n"));
803
804 //
805 // Disable the timer to trigger the process of async transfers temporarily.
806 //
807 Status = gBS->SetTimer (Private->TimerEvent, TimerCancel, 0);
808 if (EFI_ERROR (Status)) {
809 goto EXIT;
810 }
811
812 //
813 // Reset the NVMe controller.
814 //
815 Status = NvmeControllerInit (Private);
816 if (!EFI_ERROR (Status)) {
817 Status = AbortAsyncPassThruTasks (Private);
818 if (!EFI_ERROR (Status)) {
819 //
820 // Re-enable the timer to trigger the process of async transfers.
821 //
822 Status = gBS->SetTimer (Private->TimerEvent, TimerPeriodic, NVME_HC_ASYNC_TIMER);
823 if (!EFI_ERROR (Status)) {
824 //
825 // Return EFI_TIMEOUT to indicate a timeout occurs for NVMe PassThru command.
826 //
827 Status = EFI_TIMEOUT;
828 }
829 }
830 } else {
831 Status = EFI_DEVICE_ERROR;
832 }
833
834 goto EXIT;
835 }
836
837 if ((Private->CqHdbl[QueueId].Cqh ^= 1) == 0) {
838 Private->Pt[QueueId] ^= 1;
839 }
840
841 Data = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueId]);
842 PreviousStatus = Status;
843 Status = PciIo->Mem.Write (
844 PciIo,
845 EfiPciIoWidthUint32,
846 NVME_BAR,
847 NVME_CQHDBL_OFFSET(QueueId, Private->Cap.Dstrd),
848 1,
849 &Data
850 );
851 // The return status of PciIo->Mem.Write should not override
852 // previous status if previous status contains error.
853 Status = EFI_ERROR (PreviousStatus) ? PreviousStatus : Status;
854
855 //
856 // For now, the code does not support the non-blocking feature for admin queue.
857 // If Event is not NULL for admin queue, signal the caller's event here.
858 //
859 if (Event != NULL) {
860 ASSERT (QueueId == 0);
861 gBS->SignalEvent (Event);
862 }
863
864 EXIT:
865 if (MapData != NULL) {
866 PciIo->Unmap (
867 PciIo,
868 MapData
869 );
870 }
871
872 if (MapMeta != NULL) {
873 PciIo->Unmap (
874 PciIo,
875 MapMeta
876 );
877 }
878
879 if (MapPrpList != NULL) {
880 PciIo->Unmap (
881 PciIo,
882 MapPrpList
883 );
884 }
885
886 if (Prp != NULL) {
887 PciIo->FreeBuffer (PciIo, PrpListNo, PrpListHost);
888 }
889
890 if (TimerEvent != NULL) {
891 gBS->CloseEvent (TimerEvent);
892 }
893 return Status;
894 }
895
896 /**
897 Used to retrieve the next namespace ID for this NVM Express controller.
898
899 The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNextNamespace() function retrieves the next valid
900 namespace ID on this NVM Express controller.
901
902 If on input the value pointed to by NamespaceId is 0xFFFFFFFF, then the first valid namespace
903 ID defined on the NVM Express controller is returned in the location pointed to by NamespaceId
904 and a status of EFI_SUCCESS is returned.
905
906 If on input the value pointed to by NamespaceId is an invalid namespace ID other than 0xFFFFFFFF,
907 then EFI_INVALID_PARAMETER is returned.
908
909 If on input the value pointed to by NamespaceId is a valid namespace ID, then the next valid
910 namespace ID on the NVM Express controller is returned in the location pointed to by NamespaceId,
911 and EFI_SUCCESS is returned.
912
913 If the value pointed to by NamespaceId is the namespace ID of the last namespace on the NVM
914 Express controller, then EFI_NOT_FOUND is returned.
915
916 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
917 @param[in,out] NamespaceId On input, a pointer to a legal NamespaceId for an NVM Express
918 namespace present on the NVM Express controller. On output, a
919 pointer to the next NamespaceId of an NVM Express namespace on
920 an NVM Express controller. An input value of 0xFFFFFFFF retrieves
921 the first NamespaceId for an NVM Express namespace present on an
922 NVM Express controller.
923
924 @retval EFI_SUCCESS The Namespace ID of the next Namespace was returned.
925 @retval EFI_NOT_FOUND There are no more namespaces defined on this controller.
926 @retval EFI_INVALID_PARAMETER NamespaceId is an invalid value other than 0xFFFFFFFF.
927
928 **/
929 EFI_STATUS
930 EFIAPI
931 NvmExpressGetNextNamespace (
932 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *This,
933 IN OUT UINT32 *NamespaceId
934 )
935 {
936 NVME_CONTROLLER_PRIVATE_DATA *Private;
937 NVME_ADMIN_NAMESPACE_DATA *NamespaceData;
938 UINT32 NextNamespaceId;
939 EFI_STATUS Status;
940
941 if ((This == NULL) || (NamespaceId == NULL)) {
942 return EFI_INVALID_PARAMETER;
943 }
944
945 NamespaceData = NULL;
946 Status = EFI_NOT_FOUND;
947
948 Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
949 //
950 // If the NamespaceId input value is 0xFFFFFFFF, then get the first valid namespace ID
951 //
952 if (*NamespaceId == 0xFFFFFFFF) {
953 //
954 // Start with the first namespace ID
955 //
956 NextNamespaceId = 1;
957 //
958 // Allocate buffer for Identify Namespace data.
959 //
960 NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
961
962 if (NamespaceData == NULL) {
963 return EFI_NOT_FOUND;
964 }
965
966 Status = NvmeIdentifyNamespace (Private, NextNamespaceId, NamespaceData);
967 if (EFI_ERROR(Status)) {
968 goto Done;
969 }
970
971 *NamespaceId = NextNamespaceId;
972 } else {
973 if (*NamespaceId > Private->ControllerData->Nn) {
974 return EFI_INVALID_PARAMETER;
975 }
976
977 NextNamespaceId = *NamespaceId + 1;
978 if (NextNamespaceId > Private->ControllerData->Nn) {
979 return EFI_NOT_FOUND;
980 }
981
982 //
983 // Allocate buffer for Identify Namespace data.
984 //
985 NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
986 if (NamespaceData == NULL) {
987 return EFI_NOT_FOUND;
988 }
989
990 Status = NvmeIdentifyNamespace (Private, NextNamespaceId, NamespaceData);
991 if (EFI_ERROR(Status)) {
992 goto Done;
993 }
994
995 *NamespaceId = NextNamespaceId;
996 }
997
998 Done:
999 if (NamespaceData != NULL) {
1000 FreePool(NamespaceData);
1001 }
1002
1003 return Status;
1004 }
1005
1006 /**
1007 Used to translate a device path node to a namespace ID.
1008
1009 The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNamespace() function determines the namespace ID associated with the
1010 namespace described by DevicePath.
1011
1012 If DevicePath is a device path node type that the NVM Express Pass Thru driver supports, then the NVM Express
1013 Pass Thru driver will attempt to translate the contents DevicePath into a namespace ID.
1014
1015 If this translation is successful, then that namespace ID is returned in NamespaceId, and EFI_SUCCESS is returned
1016
1017 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
1018 @param[in] DevicePath A pointer to the device path node that describes an NVM Express namespace on
1019 the NVM Express controller.
1020 @param[out] NamespaceId The NVM Express namespace ID contained in the device path node.
1021
1022 @retval EFI_SUCCESS DevicePath was successfully translated to NamespaceId.
1023 @retval EFI_INVALID_PARAMETER If DevicePath or NamespaceId are NULL, then EFI_INVALID_PARAMETER is returned.
1024 @retval EFI_UNSUPPORTED If DevicePath is not a device path node type that the NVM Express Pass Thru driver
1025 supports, then EFI_UNSUPPORTED is returned.
1026 @retval EFI_NOT_FOUND If DevicePath is a device path node type that the NVM Express Pass Thru driver
1027 supports, but there is not a valid translation from DevicePath to a namespace ID,
1028 then EFI_NOT_FOUND is returned.
1029 **/
1030 EFI_STATUS
1031 EFIAPI
1032 NvmExpressGetNamespace (
1033 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *This,
1034 IN EFI_DEVICE_PATH_PROTOCOL *DevicePath,
1035 OUT UINT32 *NamespaceId
1036 )
1037 {
1038 NVME_NAMESPACE_DEVICE_PATH *Node;
1039 NVME_CONTROLLER_PRIVATE_DATA *Private;
1040
1041 if ((This == NULL) || (DevicePath == NULL) || (NamespaceId == NULL)) {
1042 return EFI_INVALID_PARAMETER;
1043 }
1044
1045 if (DevicePath->Type != MESSAGING_DEVICE_PATH) {
1046 return EFI_UNSUPPORTED;
1047 }
1048
1049 Node = (NVME_NAMESPACE_DEVICE_PATH *)DevicePath;
1050 Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
1051
1052 if (DevicePath->SubType == MSG_NVME_NAMESPACE_DP) {
1053 if (DevicePathNodeLength(DevicePath) != sizeof(NVME_NAMESPACE_DEVICE_PATH)) {
1054 return EFI_NOT_FOUND;
1055 }
1056
1057 //
1058 // Check NamespaceId in the device path node is valid or not.
1059 //
1060 if ((Node->NamespaceId == 0) ||
1061 (Node->NamespaceId > Private->ControllerData->Nn)) {
1062 return EFI_NOT_FOUND;
1063 }
1064
1065 *NamespaceId = Node->NamespaceId;
1066
1067 return EFI_SUCCESS;
1068 } else {
1069 return EFI_UNSUPPORTED;
1070 }
1071 }
1072
1073 /**
1074 Used to allocate and build a device path node for an NVM Express namespace on an NVM Express controller.
1075
1076 The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.BuildDevicePath() function allocates and builds a single device
1077 path node for the NVM Express namespace specified by NamespaceId.
1078
1079 If the NamespaceId is not valid, then EFI_NOT_FOUND is returned.
1080
1081 If DevicePath is NULL, then EFI_INVALID_PARAMETER is returned.
1082
1083 If there are not enough resources to allocate the device path node, then EFI_OUT_OF_RESOURCES is returned.
1084
1085 Otherwise, DevicePath is allocated with the boot service AllocatePool(), the contents of DevicePath are
1086 initialized to describe the NVM Express namespace specified by NamespaceId, and EFI_SUCCESS is returned.
1087
1088 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
1089 @param[in] NamespaceId The NVM Express namespace ID for which a device path node is to be
1090 allocated and built. Caller must set the NamespaceId to zero if the
1091 device path node will contain a valid UUID.
1092 @param[in,out] DevicePath A pointer to a single device path node that describes the NVM Express
1093 namespace specified by NamespaceId. This function is responsible for
1094 allocating the buffer DevicePath with the boot service AllocatePool().
1095 It is the caller's responsibility to free DevicePath when the caller
1096 is finished with DevicePath.
1097 @retval EFI_SUCCESS The device path node that describes the NVM Express namespace specified
1098 by NamespaceId was allocated and returned in DevicePath.
1099 @retval EFI_NOT_FOUND The NamespaceId is not valid.
1100 @retval EFI_INVALID_PARAMETER DevicePath is NULL.
1101 @retval EFI_OUT_OF_RESOURCES There are not enough resources to allocate the DevicePath node.
1102
1103 **/
1104 EFI_STATUS
1105 EFIAPI
1106 NvmExpressBuildDevicePath (
1107 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *This,
1108 IN UINT32 NamespaceId,
1109 IN OUT EFI_DEVICE_PATH_PROTOCOL **DevicePath
1110 )
1111 {
1112 NVME_NAMESPACE_DEVICE_PATH *Node;
1113 NVME_CONTROLLER_PRIVATE_DATA *Private;
1114 EFI_STATUS Status;
1115 NVME_ADMIN_NAMESPACE_DATA *NamespaceData;
1116
1117 //
1118 // Validate parameters
1119 //
1120 if ((This == NULL) || (DevicePath == NULL)) {
1121 return EFI_INVALID_PARAMETER;
1122 }
1123
1124 Status = EFI_SUCCESS;
1125 Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
1126
1127 //
1128 // Check NamespaceId is valid or not.
1129 //
1130 if ((NamespaceId == 0) ||
1131 (NamespaceId > Private->ControllerData->Nn)) {
1132 return EFI_NOT_FOUND;
1133 }
1134
1135 Node = (NVME_NAMESPACE_DEVICE_PATH *)AllocateZeroPool (sizeof (NVME_NAMESPACE_DEVICE_PATH));
1136 if (Node == NULL) {
1137 return EFI_OUT_OF_RESOURCES;
1138 }
1139
1140 Node->Header.Type = MESSAGING_DEVICE_PATH;
1141 Node->Header.SubType = MSG_NVME_NAMESPACE_DP;
1142 SetDevicePathNodeLength (&Node->Header, sizeof (NVME_NAMESPACE_DEVICE_PATH));
1143 Node->NamespaceId = NamespaceId;
1144
1145 //
1146 // Allocate a buffer for Identify Namespace data.
1147 //
1148 NamespaceData = NULL;
1149 NamespaceData = AllocateZeroPool(sizeof (NVME_ADMIN_NAMESPACE_DATA));
1150 if(NamespaceData == NULL) {
1151 Status = EFI_OUT_OF_RESOURCES;
1152 goto Exit;
1153 }
1154
1155 //
1156 // Get UUID from specified Identify Namespace data.
1157 //
1158 Status = NvmeIdentifyNamespace (
1159 Private,
1160 NamespaceId,
1161 (VOID *)NamespaceData
1162 );
1163
1164 if (EFI_ERROR(Status)) {
1165 goto Exit;
1166 }
1167
1168 Node->NamespaceUuid = NamespaceData->Eui64;
1169
1170 *DevicePath = (EFI_DEVICE_PATH_PROTOCOL *)Node;
1171
1172 Exit:
1173 if(NamespaceData != NULL) {
1174 FreePool (NamespaceData);
1175 }
1176
1177 if (EFI_ERROR (Status)) {
1178 FreePool (Node);
1179 }
1180
1181 return Status;
1182 }
1183