3 Generic non-coherent implementation of DmaLib.h
5 Copyright (c) 2008 - 2010, Apple Inc. All rights reserved.<BR>
6 Copyright (c) 2015 - 2017, Linaro, Ltd. All rights reserved.<BR>
8 SPDX-License-Identifier: BSD-2-Clause-Patent
13 #include <Library/BaseLib.h>
14 #include <Library/DebugLib.h>
15 #include <Library/DmaLib.h>
16 #include <Library/DxeServicesTableLib.h>
17 #include <Library/MemoryAllocationLib.h>
18 #include <Library/UefiBootServicesTableLib.h>
19 #include <Library/IoLib.h>
20 #include <Library/BaseMemoryLib.h>
22 #include <Protocol/Cpu.h>
25 EFI_PHYSICAL_ADDRESS HostAddress
;
28 DMA_MAP_OPERATION Operation
;
38 } UNCACHED_ALLOCATION
;
40 STATIC EFI_CPU_ARCH_PROTOCOL
*mCpu
;
41 STATIC LIST_ENTRY UncachedAllocationList
;
43 STATIC PHYSICAL_ADDRESS mDmaHostAddressLimit
;
51 return (PHYSICAL_ADDRESS
)(UINTN
)Address
+ PcdGet64 (PcdDmaDeviceOffset
);
55 Allocates one or more 4KB pages of a certain memory type at a specified
58 Allocates the number of 4KB pages specified by Pages of a certain memory type
59 with an alignment specified by Alignment. The allocated buffer is returned.
60 If Pages is 0, then NULL is returned. If there is not enough memory at the
61 specified alignment remaining to satisfy the request, then NULL is returned.
62 If Alignment is not a power of two and Alignment is not zero, then ASSERT().
63 If Pages plus EFI_SIZE_TO_PAGES (Alignment) overflows, then ASSERT().
65 @param MemoryType The type of memory to allocate.
66 @param Pages The number of 4 KB pages to allocate.
67 @param Alignment The requested alignment of the allocation.
68 Must be a power of two.
69 If Alignment is zero, then byte alignment is
72 @return A pointer to the allocated buffer or NULL if allocation fails.
77 InternalAllocateAlignedPages (
78 IN EFI_MEMORY_TYPE MemoryType
,
84 EFI_PHYSICAL_ADDRESS Memory
;
91 // Alignment must be a power of two or zero.
93 ASSERT ((Alignment
& (Alignment
- 1)) == 0);
98 if (Alignment
> EFI_PAGE_SIZE
) {
100 // Calculate the total number of pages since alignment is larger than page
103 AlignmentMask
= Alignment
- 1;
104 RealPages
= Pages
+ EFI_SIZE_TO_PAGES (Alignment
);
106 // Make sure that Pages plus EFI_SIZE_TO_PAGES (Alignment) does not
109 ASSERT (RealPages
> Pages
);
111 Memory
= mDmaHostAddressLimit
;
112 Status
= gBS
->AllocatePages (AllocateMaxAddress
, MemoryType
, RealPages
,
114 if (EFI_ERROR (Status
)) {
117 AlignedMemory
= ((UINTN
)Memory
+ AlignmentMask
) & ~AlignmentMask
;
118 UnalignedPages
= EFI_SIZE_TO_PAGES (AlignedMemory
- (UINTN
)Memory
);
119 if (UnalignedPages
> 0) {
121 // Free first unaligned page(s).
123 Status
= gBS
->FreePages (Memory
, UnalignedPages
);
124 ASSERT_EFI_ERROR (Status
);
126 Memory
= AlignedMemory
+ EFI_PAGES_TO_SIZE (Pages
);
127 UnalignedPages
= RealPages
- Pages
- UnalignedPages
;
128 if (UnalignedPages
> 0) {
130 // Free last unaligned page(s).
132 Status
= gBS
->FreePages (Memory
, UnalignedPages
);
133 ASSERT_EFI_ERROR (Status
);
137 // Do not over-allocate pages in this case.
139 Memory
= mDmaHostAddressLimit
;
140 Status
= gBS
->AllocatePages (AllocateMaxAddress
, MemoryType
, Pages
,
142 if (EFI_ERROR (Status
)) {
145 AlignedMemory
= (UINTN
)Memory
;
147 return (VOID
*)AlignedMemory
;
151 Provides the DMA controller-specific addresses needed to access system memory.
153 Operation is relative to the DMA bus master.
155 @param Operation Indicates if the bus master is going to read or
156 write to system memory.
157 @param HostAddress The system memory address to map to the DMA
159 @param NumberOfBytes On input the number of bytes to map. On output
160 the number of bytes that were mapped.
161 @param DeviceAddress The resulting map address for the bus master
162 controller to use to access the host's
164 @param Mapping A resulting value to pass to Unmap().
166 @retval EFI_SUCCESS The range was mapped for the returned
168 @retval EFI_UNSUPPORTED The HostAddress cannot be mapped as a common
170 @retval EFI_INVALID_PARAMETER One or more parameters are invalid.
171 @retval EFI_OUT_OF_RESOURCES The request could not be completed due to a lack
173 @retval EFI_DEVICE_ERROR The system hardware could not map the requested
180 IN DMA_MAP_OPERATION Operation
,
181 IN VOID
*HostAddress
,
182 IN OUT UINTN
*NumberOfBytes
,
183 OUT PHYSICAL_ADDRESS
*DeviceAddress
,
188 MAP_INFO_INSTANCE
*Map
;
190 EFI_GCD_MEMORY_SPACE_DESCRIPTOR GcdDescriptor
;
193 if (HostAddress
== NULL
||
194 NumberOfBytes
== NULL
||
195 DeviceAddress
== NULL
||
197 return EFI_INVALID_PARAMETER
;
200 if (Operation
>= MapOperationMaximum
) {
201 return EFI_INVALID_PARAMETER
;
204 *DeviceAddress
= HostToDeviceAddress (HostAddress
);
206 // Remember range so we can flush on the other side
207 Map
= AllocatePool (sizeof (MAP_INFO_INSTANCE
));
209 return EFI_OUT_OF_RESOURCES
;
212 if (((UINTN
)HostAddress
+ *NumberOfBytes
) > mDmaHostAddressLimit
) {
214 if (Operation
== MapOperationBusMasterCommonBuffer
) {
215 goto CommonBufferError
;
218 AllocSize
= ALIGN_VALUE (*NumberOfBytes
, mCpu
->DmaBufferAlignment
);
219 Map
->BufferAddress
= InternalAllocateAlignedPages (EfiBootServicesData
,
220 EFI_SIZE_TO_PAGES (AllocSize
),
221 mCpu
->DmaBufferAlignment
);
222 if (Map
->BufferAddress
== NULL
) {
223 Status
= EFI_OUT_OF_RESOURCES
;
227 if (Map
->Operation
== MapOperationBusMasterRead
) {
228 CopyMem (Map
->BufferAddress
, (VOID
*)(UINTN
)HostAddress
, *NumberOfBytes
);
230 mCpu
->FlushDataCache (mCpu
, (UINTN
)Map
->BufferAddress
, AllocSize
,
231 EfiCpuFlushTypeWriteBack
);
233 *DeviceAddress
= HostToDeviceAddress (Map
->BufferAddress
);
234 } else if (Operation
!= MapOperationBusMasterRead
&&
235 ((((UINTN
)HostAddress
& (mCpu
->DmaBufferAlignment
- 1)) != 0) ||
236 ((*NumberOfBytes
& (mCpu
->DmaBufferAlignment
- 1)) != 0))) {
238 // Get the cacheability of the region
239 Status
= gDS
->GetMemorySpaceDescriptor ((UINTN
)HostAddress
, &GcdDescriptor
);
240 if (EFI_ERROR(Status
)) {
244 // If the mapped buffer is not an uncached buffer
245 if ((GcdDescriptor
.Attributes
& (EFI_MEMORY_WB
| EFI_MEMORY_WT
)) != 0) {
247 // Operations of type MapOperationBusMasterCommonBuffer are only allowed
248 // on uncached buffers.
250 if (Operation
== MapOperationBusMasterCommonBuffer
) {
251 goto CommonBufferError
;
255 // If the buffer does not fill entire cache lines we must double buffer
256 // into a suitably aligned allocation that allows us to invalidate the
257 // cache without running the risk of corrupting adjacent unrelated data.
258 // Note that pool allocations are guaranteed to be 8 byte aligned, so
259 // we only have to add (alignment - 8) worth of padding.
261 Map
->DoubleBuffer
= TRUE
;
262 AllocSize
= ALIGN_VALUE (*NumberOfBytes
, mCpu
->DmaBufferAlignment
) +
263 (mCpu
->DmaBufferAlignment
- 8);
264 Map
->BufferAddress
= AllocatePool (AllocSize
);
265 if (Map
->BufferAddress
== NULL
) {
266 Status
= EFI_OUT_OF_RESOURCES
;
270 Buffer
= ALIGN_POINTER (Map
->BufferAddress
, mCpu
->DmaBufferAlignment
);
271 *DeviceAddress
= HostToDeviceAddress (Buffer
);
274 // Get rid of any dirty cachelines covering the double buffer. This
275 // prevents them from being written back unexpectedly, potentially
276 // overwriting the data we receive from the device.
278 mCpu
->FlushDataCache (mCpu
, (UINTN
)Buffer
, *NumberOfBytes
,
279 EfiCpuFlushTypeWriteBack
);
281 Map
->DoubleBuffer
= FALSE
;
284 Map
->DoubleBuffer
= FALSE
;
289 // The operation type check above only executes if the buffer happens to be
290 // misaligned with respect to CWG, but even if it is aligned, we should not
291 // allow arbitrary buffers to be used for creating consistent mappings.
292 // So duplicate the check here when running in DEBUG mode, just to assert
293 // that we are not trying to create a consistent mapping for cached memory.
295 Status
= gDS
->GetMemorySpaceDescriptor ((UINTN
)HostAddress
, &GcdDescriptor
);
296 ASSERT_EFI_ERROR(Status
);
298 ASSERT (Operation
!= MapOperationBusMasterCommonBuffer
||
299 (GcdDescriptor
.Attributes
& (EFI_MEMORY_WB
| EFI_MEMORY_WT
)) == 0);
303 // Flush the Data Cache (should not have any effect if the memory region is
305 mCpu
->FlushDataCache (mCpu
, (UINTN
)HostAddress
, *NumberOfBytes
,
306 EfiCpuFlushTypeWriteBackInvalidate
);
309 Map
->HostAddress
= (UINTN
)HostAddress
;
310 Map
->NumberOfBytes
= *NumberOfBytes
;
311 Map
->Operation
= Operation
;
319 "%a: Operation type 'MapOperationBusMasterCommonBuffer' is only "
320 "supported\non memory regions that were allocated using "
321 "DmaAllocateBuffer ()\n", __FUNCTION__
));
322 Status
= EFI_UNSUPPORTED
;
331 Completes the DmaMapBusMasterRead(), DmaMapBusMasterWrite(), or
332 DmaMapBusMasterCommonBuffer() operation and releases any corresponding
335 @param Mapping The mapping value returned from DmaMap*().
337 @retval EFI_SUCCESS The range was unmapped.
338 @retval EFI_DEVICE_ERROR The data was not committed to the target system
340 @retval EFI_INVALID_PARAMETER An inconsistency was detected between the
341 mapping type and the DoubleBuffer field
350 MAP_INFO_INSTANCE
*Map
;
355 if (Mapping
== NULL
) {
357 return EFI_INVALID_PARAMETER
;
360 Map
= (MAP_INFO_INSTANCE
*)Mapping
;
362 Status
= EFI_SUCCESS
;
363 if (((UINTN
)Map
->HostAddress
+ Map
->NumberOfBytes
) > mDmaHostAddressLimit
) {
364 AllocSize
= ALIGN_VALUE (Map
->NumberOfBytes
, mCpu
->DmaBufferAlignment
);
365 if (Map
->Operation
== MapOperationBusMasterWrite
) {
366 mCpu
->FlushDataCache (mCpu
, (UINTN
)Map
->BufferAddress
, AllocSize
,
367 EfiCpuFlushTypeInvalidate
);
368 CopyMem ((VOID
*)(UINTN
)Map
->HostAddress
, Map
->BufferAddress
,
371 FreePages (Map
->BufferAddress
, EFI_SIZE_TO_PAGES (AllocSize
));
372 } else if (Map
->DoubleBuffer
) {
374 ASSERT (Map
->Operation
== MapOperationBusMasterWrite
);
376 if (Map
->Operation
!= MapOperationBusMasterWrite
) {
377 Status
= EFI_INVALID_PARAMETER
;
379 Buffer
= ALIGN_POINTER (Map
->BufferAddress
, mCpu
->DmaBufferAlignment
);
381 mCpu
->FlushDataCache (mCpu
, (UINTN
)Buffer
, Map
->NumberOfBytes
,
382 EfiCpuFlushTypeInvalidate
);
384 CopyMem ((VOID
*)(UINTN
)Map
->HostAddress
, Buffer
, Map
->NumberOfBytes
);
386 FreePool (Map
->BufferAddress
);
389 if (Map
->Operation
== MapOperationBusMasterWrite
) {
391 // Make sure we read buffer from uncached memory and not the cache
393 mCpu
->FlushDataCache (mCpu
, Map
->HostAddress
, Map
->NumberOfBytes
,
394 EfiCpuFlushTypeInvalidate
);
404 Allocates pages that are suitable for an DmaMap() of type
405 MapOperationBusMasterCommonBuffer mapping.
407 @param MemoryType The type of memory to allocate,
408 EfiBootServicesData or EfiRuntimeServicesData.
409 @param Pages The number of pages to allocate.
410 @param HostAddress A pointer to store the base system memory
411 address of the allocated range.
413 @retval EFI_SUCCESS The requested memory pages were allocated.
414 @retval EFI_INVALID_PARAMETER One or more parameters are invalid.
415 @retval EFI_OUT_OF_RESOURCES The memory pages could not be allocated.
421 IN EFI_MEMORY_TYPE MemoryType
,
423 OUT VOID
**HostAddress
426 return DmaAllocateAlignedBuffer (MemoryType
, Pages
, 0, HostAddress
);
430 Allocates pages that are suitable for an DmaMap() of type
431 MapOperationBusMasterCommonBuffer mapping, at the requested alignment.
433 @param MemoryType The type of memory to allocate,
434 EfiBootServicesData or EfiRuntimeServicesData.
435 @param Pages The number of pages to allocate.
436 @param Alignment Alignment in bytes of the base of the returned
437 buffer (must be a power of 2)
438 @param HostAddress A pointer to store the base system memory
439 address of the allocated range.
441 @retval EFI_SUCCESS The requested memory pages were allocated.
442 @retval EFI_INVALID_PARAMETER One or more parameters are invalid.
443 @retval EFI_OUT_OF_RESOURCES The memory pages could not be allocated.
448 DmaAllocateAlignedBuffer (
449 IN EFI_MEMORY_TYPE MemoryType
,
452 OUT VOID
**HostAddress
455 EFI_GCD_MEMORY_SPACE_DESCRIPTOR GcdDescriptor
;
458 UNCACHED_ALLOCATION
*Alloc
;
461 if (Alignment
== 0) {
462 Alignment
= EFI_PAGE_SIZE
;
465 if (HostAddress
== NULL
||
466 (Alignment
& (Alignment
- 1)) != 0) {
467 return EFI_INVALID_PARAMETER
;
470 if (MemoryType
== EfiBootServicesData
||
471 MemoryType
== EfiRuntimeServicesData
) {
472 Allocation
= InternalAllocateAlignedPages (MemoryType
, Pages
, Alignment
);
474 return EFI_INVALID_PARAMETER
;
477 if (Allocation
== NULL
) {
478 return EFI_OUT_OF_RESOURCES
;
481 // Get the cacheability of the region
482 Status
= gDS
->GetMemorySpaceDescriptor ((UINTN
)Allocation
, &GcdDescriptor
);
483 if (EFI_ERROR(Status
)) {
487 // Choose a suitable uncached memory type that is supported by the region
488 if (GcdDescriptor
.Capabilities
& EFI_MEMORY_WC
) {
489 MemType
= EFI_MEMORY_WC
;
490 } else if (GcdDescriptor
.Capabilities
& EFI_MEMORY_UC
) {
491 MemType
= EFI_MEMORY_UC
;
493 Status
= EFI_UNSUPPORTED
;
497 Alloc
= AllocatePool (sizeof *Alloc
);
502 Alloc
->HostAddress
= Allocation
;
503 Alloc
->NumPages
= Pages
;
504 Alloc
->Attributes
= GcdDescriptor
.Attributes
;
506 InsertHeadList (&UncachedAllocationList
, &Alloc
->Link
);
508 // Remap the region with the new attributes
509 Status
= gDS
->SetMemorySpaceAttributes ((PHYSICAL_ADDRESS
)(UINTN
)Allocation
,
510 EFI_PAGES_TO_SIZE (Pages
),
512 if (EFI_ERROR (Status
)) {
516 Status
= mCpu
->FlushDataCache (mCpu
,
517 (PHYSICAL_ADDRESS
)(UINTN
)Allocation
,
518 EFI_PAGES_TO_SIZE (Pages
),
519 EfiCpuFlushTypeInvalidate
);
520 if (EFI_ERROR (Status
)) {
524 *HostAddress
= Allocation
;
529 RemoveEntryList (&Alloc
->Link
);
533 FreePages (Allocation
, Pages
);
539 Frees memory that was allocated with DmaAllocateBuffer().
541 @param Pages The number of pages to free.
542 @param HostAddress The base system memory address of the allocated
545 @retval EFI_SUCCESS The requested memory pages were freed.
546 @retval EFI_INVALID_PARAMETER The memory range specified by HostAddress and
547 Pages was not allocated with
559 UNCACHED_ALLOCATION
*Alloc
;
563 if (HostAddress
== NULL
) {
564 return EFI_INVALID_PARAMETER
;
567 for (Link
= GetFirstNode (&UncachedAllocationList
), Found
= FALSE
;
568 !IsNull (&UncachedAllocationList
, Link
);
569 Link
= GetNextNode (&UncachedAllocationList
, Link
)) {
571 Alloc
= BASE_CR (Link
, UNCACHED_ALLOCATION
, Link
);
572 if (Alloc
->HostAddress
== HostAddress
&& Alloc
->NumPages
== Pages
) {
580 return EFI_INVALID_PARAMETER
;
583 RemoveEntryList (&Alloc
->Link
);
585 Status
= gDS
->SetMemorySpaceAttributes ((PHYSICAL_ADDRESS
)(UINTN
)HostAddress
,
586 EFI_PAGES_TO_SIZE (Pages
),
588 if (EFI_ERROR (Status
)) {
593 // If we fail to restore the original attributes, it is better to leak the
594 // memory than to return it to the heap
596 FreePages (HostAddress
, Pages
);
606 NonCoherentDmaLibConstructor (
607 IN EFI_HANDLE ImageHandle
,
608 IN EFI_SYSTEM_TABLE
*SystemTable
611 InitializeListHead (&UncachedAllocationList
);
614 // Ensure that the combination of DMA addressing offset and limit produces
617 ASSERT (PcdGet64 (PcdDmaDeviceLimit
) > PcdGet64 (PcdDmaDeviceOffset
));
619 mDmaHostAddressLimit
= PcdGet64 (PcdDmaDeviceLimit
) -
620 PcdGet64 (PcdDmaDeviceOffset
);
622 // Get the Cpu protocol for later use
623 return gBS
->LocateProtocol (&gEfiCpuArchProtocolGuid
, NULL
, (VOID
**)&mCpu
);