3 Copyright (c) 2008-2009, Apple Inc. All rights reserved.
5 All rights reserved. This program and the accompanying materials
6 are licensed and made available under the terms and conditions of the BSD License
7 which accompanies this distribution. The full text of the license may be found at
8 http://opensource.org/licenses/bsd-license.php
10 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
15 #include <Library/BaseMemoryLib.h>
16 #include <Library/DebugLib.h>
17 #include <Library/UefiBootServicesTableLib.h>
18 #include <Library/UncachedMemoryAllocationLib.h>
19 #include <Library/PcdLib.h>
20 #include <Library/ArmLib.h>
21 #include <Protocol/Cpu.h>
24 ConvertToPhysicalAddress (
25 IN VOID
*VirtualAddress
28 UINTN UncachedMemoryMask
= (UINTN
)PcdGet64(PcdArmUncachedMemoryMask
);
29 UINTN PhysicalAddress
;
31 PhysicalAddress
= (UINTN
)VirtualAddress
& ~UncachedMemoryMask
;
33 return (EFI_PHYSICAL_ADDRESS
)PhysicalAddress
;
37 ConvertToCachedAddress (
41 return (VOID
*)(UINTN
)ConvertToPhysicalAddress(Address
);
45 ConvertToUncachedAddress (
49 UINTN UncachedMemoryMask
= (UINTN
)PcdGet64(PcdArmUncachedMemoryMask
);
50 UINTN UncachedAddress
;
52 UncachedAddress
= (UINTN
)Address
| UncachedMemoryMask
;
54 return (VOID
*)UncachedAddress
;
59 IN EFI_PHYSICAL_ADDRESS Address
,
63 EFI_CPU_ARCH_PROTOCOL
*Cpu
;
66 Status
= gBS
->LocateProtocol(&gEfiCpuArchProtocolGuid
, NULL
, (VOID
**)&Cpu
);
67 ASSERT_EFI_ERROR(Status
);
69 Status
= Cpu
->FlushDataCache(Cpu
, Address
, Size
, EfiCpuFlushTypeWriteBackInvalidate
);
70 ASSERT_EFI_ERROR(Status
);
74 UncachedInternalAllocatePages (
75 IN EFI_MEMORY_TYPE MemoryType
,
80 EFI_PHYSICAL_ADDRESS Memory
;
86 Status
= gBS
->AllocatePages (AllocateAnyPages
, MemoryType
, Pages
, &Memory
);
87 if (EFI_ERROR (Status
)) {
92 FlushCache(Memory
, EFI_PAGES_TO_SIZE(Pages
));
93 Memory
= (EFI_PHYSICAL_ADDRESS
)(UINTN
)ConvertToUncachedAddress((VOID
*)(UINTN
)Memory
);
96 return (VOID
*) (UINTN
) Memory
;
101 UncachedAllocatePages (
105 return UncachedInternalAllocatePages (EfiBootServicesData
, Pages
);
110 UncachedAllocateRuntimePages (
114 return UncachedInternalAllocatePages (EfiRuntimeServicesData
, Pages
);
119 UncachedAllocateReservedPages (
123 return UncachedInternalAllocatePages (EfiReservedMemoryType
, Pages
);
137 Buffer
= ConvertToCachedAddress(Buffer
);
139 Status
= gBS
->FreePages ((EFI_PHYSICAL_ADDRESS
) (UINTN
) Buffer
, Pages
);
140 ASSERT_EFI_ERROR (Status
);
144 UncachedInternalAllocateAlignedPages (
145 IN EFI_MEMORY_TYPE MemoryType
,
151 EFI_PHYSICAL_ADDRESS Memory
;
154 UINTN UnalignedPages
;
158 // Alignment must be a power of two or zero.
160 ASSERT ((Alignment
& (Alignment
- 1)) == 0);
165 if (Alignment
> EFI_PAGE_SIZE
) {
167 // Caculate the total number of pages since alignment is larger than page size.
169 AlignmentMask
= Alignment
- 1;
170 RealPages
= Pages
+ EFI_SIZE_TO_PAGES (Alignment
);
172 // Make sure that Pages plus EFI_SIZE_TO_PAGES (Alignment) does not overflow.
174 ASSERT (RealPages
> Pages
);
176 Status
= gBS
->AllocatePages (AllocateAnyPages
, MemoryType
, RealPages
, &Memory
);
177 if (EFI_ERROR (Status
)) {
180 AlignedMemory
= ((UINTN
) Memory
+ AlignmentMask
) & ~AlignmentMask
;
181 UnalignedPages
= EFI_SIZE_TO_PAGES (AlignedMemory
- (UINTN
) Memory
);
182 if (UnalignedPages
> 0) {
184 // Free first unaligned page(s).
186 Status
= gBS
->FreePages (Memory
, UnalignedPages
);
187 ASSERT_EFI_ERROR (Status
);
189 Memory
= (EFI_PHYSICAL_ADDRESS
) (AlignedMemory
+ EFI_PAGES_TO_SIZE (Pages
));
190 UnalignedPages
= RealPages
- Pages
- UnalignedPages
;
191 if (UnalignedPages
> 0) {
193 // Free last unaligned page(s).
195 Status
= gBS
->FreePages (Memory
, UnalignedPages
);
196 ASSERT_EFI_ERROR (Status
);
200 // Do not over-allocate pages in this case.
202 Status
= gBS
->AllocatePages (AllocateAnyPages
, MemoryType
, Pages
, &Memory
);
203 if (EFI_ERROR (Status
)) {
206 AlignedMemory
= (UINTN
) Memory
;
209 if (AlignedMemory
!= 0) {
210 FlushCache(AlignedMemory
, EFI_PAGES_TO_SIZE(Pages
));
211 AlignedMemory
= (UINTN
)ConvertToUncachedAddress((VOID
*)AlignedMemory
);
214 return (VOID
*) AlignedMemory
;
219 UncachedAllocateAlignedPages (
224 return UncachedInternalAllocateAlignedPages (EfiBootServicesData
, Pages
, Alignment
);
229 UncachedAllocateAlignedRuntimePages (
234 return UncachedInternalAllocateAlignedPages (EfiRuntimeServicesData
, Pages
, Alignment
);
239 UncachedAllocateAlignedReservedPages (
244 return UncachedInternalAllocateAlignedPages (EfiReservedMemoryType
, Pages
, Alignment
);
249 UncachedFreeAlignedPages (
258 Buffer
= ConvertToCachedAddress(Buffer
);
260 Status
= gBS
->FreePages ((EFI_PHYSICAL_ADDRESS
) (UINTN
) Buffer
, Pages
);
261 ASSERT_EFI_ERROR (Status
);
265 UncachedInternalAllocateAlignedPool (
266 IN EFI_MEMORY_TYPE PoolType
,
267 IN UINTN AllocationSize
,
272 UINTN AlignedAddress
;
274 UINTN OverAllocationSize
;
275 UINTN RealAllocationSize
;
277 UINTN DataCacheLineLength
;
281 // Alignment must be a power of two or zero.
283 ASSERT ((Alignment
& (Alignment
- 1)) == 0);
285 DataCacheLineLength
= ArmDataCacheLineLength();
287 // Alignment must be at least cache-line aligned
288 if (Alignment
< DataCacheLineLength
) {
289 Alignment
= DataCacheLineLength
;
292 if (Alignment
== 0) {
293 AlignmentMask
= Alignment
;
295 AlignmentMask
= Alignment
- 1;
299 // Calculate the extra memory size, over-allocate memory pool and get the aligned memory address.
301 OverAllocationSize
= sizeof (RawAddress
) + AlignmentMask
;
302 RealAllocationSize
= AllocationSize
+ OverAllocationSize
;
304 // Make sure that AllocationSize plus OverAllocationSize does not overflow.
306 ASSERT (RealAllocationSize
> AllocationSize
);
308 Status
= gBS
->AllocatePool (PoolType
, RealAllocationSize
, &RawAddress
);
309 if (EFI_ERROR (Status
)) {
313 AlignedAddress
= ((UINTN
) RawAddress
+ OverAllocationSize
) & ~AlignmentMask
;
315 // Save the original memory address just before the aligned address.
317 FreePointer
= (VOID
**)(AlignedAddress
- sizeof (RawAddress
));
318 *FreePointer
= RawAddress
;
320 if (AlignedAddress
!= 0) {
321 FlushCache(AlignedAddress
, AllocationSize
);
322 AlignedAddress
= (UINTN
)ConvertToUncachedAddress((VOID
*)AlignedAddress
);
325 return (VOID
*) AlignedAddress
;
330 UncachedAllocateAlignedPool (
331 IN UINTN AllocationSize
,
335 return UncachedInternalAllocateAlignedPool (EfiBootServicesData
, AllocationSize
, Alignment
);
340 UncachedAllocateAlignedRuntimePool (
341 IN UINTN AllocationSize
,
345 return UncachedInternalAllocateAlignedPool (EfiRuntimeServicesData
, AllocationSize
, Alignment
);
350 UncachedAllocateAlignedReservedPool (
351 IN UINTN AllocationSize
,
355 return UncachedInternalAllocateAlignedPool (EfiReservedMemoryType
, AllocationSize
, Alignment
);
359 UncachedInternalAllocateAlignedZeroPool (
360 IN EFI_MEMORY_TYPE PoolType
,
361 IN UINTN AllocationSize
,
366 Memory
= UncachedInternalAllocateAlignedPool (PoolType
, AllocationSize
, Alignment
);
367 if (Memory
!= NULL
) {
368 Memory
= ZeroMem (Memory
, AllocationSize
);
375 UncachedAllocateAlignedZeroPool (
376 IN UINTN AllocationSize
,
380 return UncachedInternalAllocateAlignedZeroPool (EfiBootServicesData
, AllocationSize
, Alignment
);
385 UncachedAllocateAlignedRuntimeZeroPool (
386 IN UINTN AllocationSize
,
390 return UncachedInternalAllocateAlignedZeroPool (EfiRuntimeServicesData
, AllocationSize
, Alignment
);
395 UncachedAllocateAlignedReservedZeroPool (
396 IN UINTN AllocationSize
,
400 return UncachedInternalAllocateAlignedZeroPool (EfiReservedMemoryType
, AllocationSize
, Alignment
);
404 UncachedInternalAllocateAlignedCopyPool (
405 IN EFI_MEMORY_TYPE PoolType
,
406 IN UINTN AllocationSize
,
407 IN CONST VOID
*Buffer
,
413 ASSERT (Buffer
!= NULL
);
414 ASSERT (AllocationSize
<= (MAX_ADDRESS
- (UINTN
) Buffer
+ 1));
416 Memory
= UncachedInternalAllocateAlignedPool (PoolType
, AllocationSize
, Alignment
);
417 if (Memory
!= NULL
) {
418 Memory
= CopyMem (Memory
, Buffer
, AllocationSize
);
425 UncachedAllocateAlignedCopyPool (
426 IN UINTN AllocationSize
,
427 IN CONST VOID
*Buffer
,
431 return UncachedInternalAllocateAlignedCopyPool (EfiBootServicesData
, AllocationSize
, Buffer
, Alignment
);
436 UncachedAllocateAlignedRuntimeCopyPool (
437 IN UINTN AllocationSize
,
438 IN CONST VOID
*Buffer
,
442 return UncachedInternalAllocateAlignedCopyPool (EfiRuntimeServicesData
, AllocationSize
, Buffer
, Alignment
);
447 UncachedAllocateAlignedReservedCopyPool (
448 IN UINTN AllocationSize
,
449 IN CONST VOID
*Buffer
,
453 return UncachedInternalAllocateAlignedCopyPool (EfiReservedMemoryType
, AllocationSize
, Buffer
, Alignment
);
458 UncachedFreeAlignedPool (
466 Buffer
= ConvertToCachedAddress(Buffer
);
469 // Get the pre-saved original address in the over-allocate pool.
471 FreePointer
= (VOID
**)((UINTN
) Buffer
- sizeof (RawAddress
));
472 RawAddress
= *FreePointer
;
474 Status
= gBS
->FreePool (RawAddress
);
475 ASSERT_EFI_ERROR (Status
);
479 UncachedInternalAllocatePool (
480 IN EFI_MEMORY_TYPE MemoryType
,
481 IN UINTN AllocationSize
484 UINTN CacheLineLength
= ArmDataCacheLineLength();
485 return UncachedInternalAllocateAlignedPool(MemoryType
, AllocationSize
, CacheLineLength
);
490 UncachedAllocatePool (
491 IN UINTN AllocationSize
494 return UncachedInternalAllocatePool (EfiBootServicesData
, AllocationSize
);
499 UncachedAllocateRuntimePool (
500 IN UINTN AllocationSize
503 return UncachedInternalAllocatePool (EfiRuntimeServicesData
, AllocationSize
);
508 UncachedAllocateReservedPool (
509 IN UINTN AllocationSize
512 return UncachedInternalAllocatePool (EfiReservedMemoryType
, AllocationSize
);
516 UncachedInternalAllocateZeroPool (
517 IN EFI_MEMORY_TYPE PoolType
,
518 IN UINTN AllocationSize
523 Memory
= UncachedInternalAllocatePool (PoolType
, AllocationSize
);
524 if (Memory
!= NULL
) {
525 Memory
= ZeroMem (Memory
, AllocationSize
);
532 UncachedAllocateZeroPool (
533 IN UINTN AllocationSize
536 return UncachedInternalAllocateZeroPool (EfiBootServicesData
, AllocationSize
);
541 UncachedAllocateRuntimeZeroPool (
542 IN UINTN AllocationSize
545 return UncachedInternalAllocateZeroPool (EfiRuntimeServicesData
, AllocationSize
);
550 UncachedAllocateReservedZeroPool (
551 IN UINTN AllocationSize
554 return UncachedInternalAllocateZeroPool (EfiReservedMemoryType
, AllocationSize
);
558 UncachedInternalAllocateCopyPool (
559 IN EFI_MEMORY_TYPE PoolType
,
560 IN UINTN AllocationSize
,
561 IN CONST VOID
*Buffer
566 ASSERT (Buffer
!= NULL
);
567 ASSERT (AllocationSize
<= (MAX_ADDRESS
- (UINTN
) Buffer
+ 1));
569 Memory
= UncachedInternalAllocatePool (PoolType
, AllocationSize
);
570 if (Memory
!= NULL
) {
571 Memory
= CopyMem (Memory
, Buffer
, AllocationSize
);
578 UncachedAllocateCopyPool (
579 IN UINTN AllocationSize
,
580 IN CONST VOID
*Buffer
583 return UncachedInternalAllocateCopyPool (EfiBootServicesData
, AllocationSize
, Buffer
);
588 UncachedAllocateRuntimeCopyPool (
589 IN UINTN AllocationSize
,
590 IN CONST VOID
*Buffer
593 return UncachedInternalAllocateCopyPool (EfiRuntimeServicesData
, AllocationSize
, Buffer
);
598 UncachedAllocateReservedCopyPool (
599 IN UINTN AllocationSize
,
600 IN CONST VOID
*Buffer
603 return UncachedInternalAllocateCopyPool (EfiReservedMemoryType
, AllocationSize
, Buffer
);
612 UncachedFreeAlignedPool(Buffer
);
617 UncachedSafeFreePool (
621 if (Buffer
!= NULL
) {
622 UncachedFreePool (Buffer
);