2 UEFI Memory pool management functions.
4 Copyright (c) 2006 - 2018, Intel Corporation. All rights reserved.<BR>
5 This program and the accompanying materials
6 are licensed and made available under the terms and conditions of the BSD License
7 which accompanies this distribution. The full text of the license may be found at
8 http://opensource.org/licenses/bsd-license.php
10 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
17 #include "HeapGuard.h"
19 STATIC EFI_LOCK mPoolMemoryLock
= EFI_INITIALIZE_LOCK_VARIABLE (TPL_NOTIFY
);
21 #define POOL_FREE_SIGNATURE SIGNATURE_32('p','f','r','0')
29 #define POOL_HEAD_SIGNATURE SIGNATURE_32('p','h','d','0')
30 #define POOLPAGE_HEAD_SIGNATURE SIGNATURE_32('p','h','d','1')
39 #define SIZE_OF_POOL_HEAD OFFSET_OF(POOL_HEAD,Data)
41 #define POOL_TAIL_SIGNATURE SIGNATURE_32('p','t','a','l')
48 #define POOL_OVERHEAD (SIZE_OF_POOL_HEAD + sizeof(POOL_TAIL))
50 #define HEAD_TO_TAIL(a) \
51 ((POOL_TAIL *) (((CHAR8 *) (a)) + (a)->Size - sizeof(POOL_TAIL)));
54 // Each element is the sum of the 2 previous ones: this allows us to migrate
55 // blocks between bins by splitting them up, while not wasting too much memory
56 // as we would in a strict power-of-2 sequence
58 STATIC CONST UINT16 mPoolSizeTable
[] = {
59 128, 256, 384, 640, 1024, 1664, 2688, 4352, 7040, 11392, 18432, 29824
62 #define SIZE_TO_LIST(a) (GetPoolIndexFromSize (a))
63 #define LIST_TO_SIZE(a) (mPoolSizeTable [a])
65 #define MAX_POOL_LIST (ARRAY_SIZE (mPoolSizeTable))
67 #define MAX_POOL_SIZE (MAX_ADDRESS - POOL_OVERHEAD)
73 #define POOL_SIGNATURE SIGNATURE_32('p','l','s','t')
77 EFI_MEMORY_TYPE MemoryType
;
78 LIST_ENTRY FreeList
[MAX_POOL_LIST
];
83 // Pool header for each memory type.
85 POOL mPoolHead
[EfiMaxMemoryType
];
88 // List of pool header to search for the appropriate memory type.
90 LIST_ENTRY mPoolHeadList
= INITIALIZE_LIST_HEAD_VARIABLE (mPoolHeadList
);
93 Get pool size table index from the specified size.
95 @param Size The specified size to get index from pool table.
97 @return The index of pool size table.
102 GetPoolIndexFromSize (
108 for (Index
= 0; Index
< MAX_POOL_LIST
; Index
++) {
109 if (mPoolSizeTable
[Index
] >= Size
) {
113 return MAX_POOL_LIST
;
117 Called to initialize the pool.
128 for (Type
=0; Type
< EfiMaxMemoryType
; Type
++) {
129 mPoolHead
[Type
].Signature
= 0;
130 mPoolHead
[Type
].Used
= 0;
131 mPoolHead
[Type
].MemoryType
= (EFI_MEMORY_TYPE
) Type
;
132 for (Index
=0; Index
< MAX_POOL_LIST
; Index
++) {
133 InitializeListHead (&mPoolHead
[Type
].FreeList
[Index
]);
140 Look up pool head for specified memory type.
142 @param MemoryType Memory type of which pool head is looked for
144 @return Pointer of Corresponding pool head.
149 IN EFI_MEMORY_TYPE MemoryType
156 if ((UINT32
)MemoryType
< EfiMaxMemoryType
) {
157 return &mPoolHead
[MemoryType
];
161 // MemoryType values in the range 0x80000000..0xFFFFFFFF are reserved for use by UEFI
162 // OS loaders that are provided by operating system vendors.
163 // MemoryType values in the range 0x70000000..0x7FFFFFFF are reserved for OEM use.
165 if ((UINT32
) MemoryType
>= MEMORY_TYPE_OEM_RESERVED_MIN
) {
167 for (Link
= mPoolHeadList
.ForwardLink
; Link
!= &mPoolHeadList
; Link
= Link
->ForwardLink
) {
168 Pool
= CR(Link
, POOL
, Link
, POOL_SIGNATURE
);
169 if (Pool
->MemoryType
== MemoryType
) {
174 Pool
= CoreAllocatePoolI (EfiBootServicesData
, sizeof (POOL
), FALSE
);
179 Pool
->Signature
= POOL_SIGNATURE
;
181 Pool
->MemoryType
= MemoryType
;
182 for (Index
=0; Index
< MAX_POOL_LIST
; Index
++) {
183 InitializeListHead (&Pool
->FreeList
[Index
]);
186 InsertHeadList (&mPoolHeadList
, &Pool
->Link
);
197 Allocate pool of a particular type.
199 @param PoolType Type of pool to allocate
200 @param Size The amount of pool to allocate
201 @param Buffer The address to return a pointer to the allocated
204 @retval EFI_INVALID_PARAMETER Buffer is NULL.
205 PoolType is in the range EfiMaxMemoryType..0x6FFFFFFF.
206 PoolType is EfiPersistentMemory.
207 @retval EFI_OUT_OF_RESOURCES Size exceeds max pool size or allocation failed.
208 @retval EFI_SUCCESS Pool successfully allocated.
213 CoreInternalAllocatePool (
214 IN EFI_MEMORY_TYPE PoolType
,
223 // If it's not a valid type, fail it
225 if ((PoolType
>= EfiMaxMemoryType
&& PoolType
< MEMORY_TYPE_OEM_RESERVED_MIN
) ||
226 (PoolType
== EfiConventionalMemory
) || (PoolType
== EfiPersistentMemory
)) {
227 return EFI_INVALID_PARAMETER
;
230 if (Buffer
== NULL
) {
231 return EFI_INVALID_PARAMETER
;
237 // If size is too large, fail it
238 // Base on the EFI spec, return status of EFI_OUT_OF_RESOURCES
240 if (Size
> MAX_POOL_SIZE
) {
241 return EFI_OUT_OF_RESOURCES
;
244 NeedGuard
= IsPoolTypeToGuard (PoolType
) && !mOnGuarding
;
247 // Acquire the memory lock and make the allocation
249 Status
= CoreAcquireLockOrFail (&mPoolMemoryLock
);
250 if (EFI_ERROR (Status
)) {
251 return EFI_OUT_OF_RESOURCES
;
254 *Buffer
= CoreAllocatePoolI (PoolType
, Size
, NeedGuard
);
255 CoreReleaseLock (&mPoolMemoryLock
);
256 return (*Buffer
!= NULL
) ? EFI_SUCCESS
: EFI_OUT_OF_RESOURCES
;
260 Allocate pool of a particular type.
262 @param PoolType Type of pool to allocate
263 @param Size The amount of pool to allocate
264 @param Buffer The address to return a pointer to the allocated
267 @retval EFI_INVALID_PARAMETER Buffer is NULL.
268 PoolType is in the range EfiMaxMemoryType..0x6FFFFFFF.
269 PoolType is EfiPersistentMemory.
270 @retval EFI_OUT_OF_RESOURCES Size exceeds max pool size or allocation failed.
271 @retval EFI_SUCCESS Pool successfully allocated.
277 IN EFI_MEMORY_TYPE PoolType
,
284 Status
= CoreInternalAllocatePool (PoolType
, Size
, Buffer
);
285 if (!EFI_ERROR (Status
)) {
287 (EFI_PHYSICAL_ADDRESS
) (UINTN
) RETURN_ADDRESS (0),
288 MemoryProfileActionAllocatePool
,
294 InstallMemoryAttributesTableOnMemoryAllocation (PoolType
);
300 Internal function. Used by the pool functions to allocate pages
301 to back pool allocation requests.
303 @param PoolType The type of memory for the new pool pages
304 @param NoPages No of pages to allocate
305 @param Granularity Bits to align.
306 @param NeedGuard Flag to indicate Guard page is needed or not
308 @return The allocated memory, or NULL
313 CoreAllocatePoolPagesI (
314 IN EFI_MEMORY_TYPE PoolType
,
316 IN UINTN Granularity
,
323 Status
= CoreAcquireLockOrFail (&gMemoryLock
);
324 if (EFI_ERROR (Status
)) {
328 Buffer
= CoreAllocatePoolPages (PoolType
, NoPages
, Granularity
, NeedGuard
);
329 CoreReleaseMemoryLock ();
331 if (Buffer
!= NULL
) {
333 SetGuardForMemory ((EFI_PHYSICAL_ADDRESS
)(UINTN
)Buffer
, NoPages
);
335 ApplyMemoryProtectionPolicy(EfiConventionalMemory
, PoolType
,
336 (EFI_PHYSICAL_ADDRESS
)(UINTN
)Buffer
, EFI_PAGES_TO_SIZE (NoPages
));
342 Internal function to allocate pool of a particular type.
343 Caller must have the memory lock held
345 @param PoolType Type of pool to allocate
346 @param Size The amount of pool to allocate
347 @param NeedGuard Flag to indicate Guard page is needed or not
349 @return The allocate pool, or NULL
354 IN EFI_MEMORY_TYPE PoolType
,
367 UINTN Offset
, MaxOffset
;
373 ASSERT_LOCKED (&mPoolMemoryLock
);
375 if (PoolType
== EfiACPIReclaimMemory
||
376 PoolType
== EfiACPIMemoryNVS
||
377 PoolType
== EfiRuntimeServicesCode
||
378 PoolType
== EfiRuntimeServicesData
) {
380 Granularity
= RUNTIME_PAGE_ALLOCATION_GRANULARITY
;
382 Granularity
= DEFAULT_PAGE_ALLOCATION_GRANULARITY
;
386 // Adjust the size by the pool header & tail overhead
389 HasPoolTail
= !(NeedGuard
&&
390 ((PcdGet8 (PcdHeapGuardPropertyMask
) & BIT7
) == 0));
391 PageAsPool
= (IsHeapGuardEnabled (GUARD_HEAP_TYPE_FREED
) && !mOnGuarding
);
394 // Adjusting the Size to be of proper alignment so that
395 // we don't get an unaligned access fault later when
396 // pool_Tail is being initialized
398 Size
= ALIGN_VARIABLE (Size
);
400 Size
+= POOL_OVERHEAD
;
401 Index
= SIZE_TO_LIST(Size
);
402 Pool
= LookupPoolHead (PoolType
);
409 // If allocation is over max size, just allocate pages for the request
412 if (Index
>= SIZE_TO_LIST (Granularity
) || NeedGuard
|| PageAsPool
) {
414 Size
-= sizeof (POOL_TAIL
);
416 NoPages
= EFI_SIZE_TO_PAGES (Size
) + EFI_SIZE_TO_PAGES (Granularity
) - 1;
417 NoPages
&= ~(UINTN
)(EFI_SIZE_TO_PAGES (Granularity
) - 1);
418 Head
= CoreAllocatePoolPagesI (PoolType
, NoPages
, Granularity
, NeedGuard
);
420 Head
= AdjustPoolHeadA ((EFI_PHYSICAL_ADDRESS
)(UINTN
)Head
, NoPages
, Size
);
426 // If there's no free pool in the proper list size, go get some more pages
428 if (IsListEmpty (&Pool
->FreeList
[Index
])) {
430 Offset
= LIST_TO_SIZE (Index
);
431 MaxOffset
= Granularity
;
434 // Check the bins holding larger blocks, and carve one up if needed
436 while (++Index
< SIZE_TO_LIST (Granularity
)) {
437 if (!IsListEmpty (&Pool
->FreeList
[Index
])) {
438 Free
= CR (Pool
->FreeList
[Index
].ForwardLink
, POOL_FREE
, Link
, POOL_FREE_SIGNATURE
);
439 RemoveEntryList (&Free
->Link
);
440 NewPage
= (VOID
*) Free
;
441 MaxOffset
= LIST_TO_SIZE (Index
);
449 NewPage
= CoreAllocatePoolPagesI (PoolType
, EFI_SIZE_TO_PAGES (Granularity
),
450 Granularity
, NeedGuard
);
451 if (NewPage
== NULL
) {
456 // Serve the allocation request from the head of the allocated block
459 Head
= (POOL_HEAD
*) NewPage
;
462 // Carve up remaining space into free pool blocks
465 while (Offset
< MaxOffset
) {
466 ASSERT (Index
< MAX_POOL_LIST
);
467 FSize
= LIST_TO_SIZE(Index
);
469 while (Offset
+ FSize
<= MaxOffset
) {
470 Free
= (POOL_FREE
*) &NewPage
[Offset
];
471 Free
->Signature
= POOL_FREE_SIGNATURE
;
472 Free
->Index
= (UINT32
)Index
;
473 InsertHeadList (&Pool
->FreeList
[Index
], &Free
->Link
);
479 ASSERT (Offset
== MaxOffset
);
484 // Remove entry from free pool list
486 Free
= CR (Pool
->FreeList
[Index
].ForwardLink
, POOL_FREE
, Link
, POOL_FREE_SIGNATURE
);
487 RemoveEntryList (&Free
->Link
);
489 Head
= (POOL_HEAD
*) Free
;
497 // Account the allocation
502 // If we have a pool buffer, fill in the header & tail info
504 Head
->Signature
= (PageAsPool
) ? POOLPAGE_HEAD_SIGNATURE
: POOL_HEAD_SIGNATURE
;
506 Head
->Type
= (EFI_MEMORY_TYPE
) PoolType
;
510 Tail
= HEAD_TO_TAIL (Head
);
511 Tail
->Signature
= POOL_TAIL_SIGNATURE
;
514 Size
-= POOL_OVERHEAD
;
516 Size
-= SIZE_OF_POOL_HEAD
;
519 DEBUG_CLEAR_MEMORY (Buffer
, Size
);
523 "AllocatePoolI: Type %x, Addr %p (len %lx) %,ld\n", PoolType
,
531 DEBUG ((DEBUG_ERROR
| DEBUG_POOL
, "AllocatePool: failed to allocate %ld bytes\n", (UINT64
) Size
));
542 @param Buffer The allocated pool entry to free
543 @param PoolType Pointer to pool type
545 @retval EFI_INVALID_PARAMETER Buffer is not a valid value.
546 @retval EFI_SUCCESS Pool successfully freed.
551 CoreInternalFreePool (
553 OUT EFI_MEMORY_TYPE
*PoolType OPTIONAL
558 if (Buffer
== NULL
) {
559 return EFI_INVALID_PARAMETER
;
562 CoreAcquireLock (&mPoolMemoryLock
);
563 Status
= CoreFreePoolI (Buffer
, PoolType
);
564 CoreReleaseLock (&mPoolMemoryLock
);
571 @param Buffer The allocated pool entry to free
573 @retval EFI_INVALID_PARAMETER Buffer is not a valid value.
574 @retval EFI_SUCCESS Pool successfully freed.
584 EFI_MEMORY_TYPE PoolType
;
586 Status
= CoreInternalFreePool (Buffer
, &PoolType
);
587 if (!EFI_ERROR (Status
)) {
589 (EFI_PHYSICAL_ADDRESS
) (UINTN
) RETURN_ADDRESS (0),
590 MemoryProfileActionFreePool
,
596 InstallMemoryAttributesTableOnMemoryAllocation (PoolType
);
602 Internal function. Frees pool pages allocated via CoreAllocatePoolPagesI().
604 @param PoolType The type of memory for the pool pages
605 @param Memory The base address to free
606 @param NoPages The number of pages to free
612 IN EFI_MEMORY_TYPE PoolType
,
613 IN EFI_PHYSICAL_ADDRESS Memory
,
617 CoreAcquireMemoryLock ();
618 CoreFreePoolPages (Memory
, NoPages
);
619 CoreReleaseMemoryLock ();
621 GuardFreedPagesChecked (Memory
, NoPages
);
622 ApplyMemoryProtectionPolicy (PoolType
, EfiConventionalMemory
,
623 (EFI_PHYSICAL_ADDRESS
)(UINTN
)Memory
, EFI_PAGES_TO_SIZE (NoPages
));
627 Internal function. Frees guarded pool pages.
629 @param PoolType The type of memory for the pool pages
630 @param Memory The base address to free
631 @param NoPages The number of pages to free
636 CoreFreePoolPagesWithGuard (
637 IN EFI_MEMORY_TYPE PoolType
,
638 IN EFI_PHYSICAL_ADDRESS Memory
,
642 EFI_PHYSICAL_ADDRESS MemoryGuarded
;
643 UINTN NoPagesGuarded
;
645 MemoryGuarded
= Memory
;
646 NoPagesGuarded
= NoPages
;
648 AdjustMemoryF (&Memory
, &NoPages
);
650 // It's safe to unset Guard page inside memory lock because there should
651 // be no memory allocation occurred in updating memory page attribute at
652 // this point. And unsetting Guard page before free will prevent Guard
653 // page just freed back to pool from being allocated right away before
654 // marking it usable (from non-present to present).
656 UnsetGuardForMemory (MemoryGuarded
, NoPagesGuarded
);
658 CoreFreePoolPagesI (PoolType
, Memory
, NoPages
);
663 Internal function to free a pool entry.
664 Caller must have the memory lock held
666 @param Buffer The allocated pool entry to free
667 @param PoolType Pointer to pool type
669 @retval EFI_INVALID_PARAMETER Buffer not valid
670 @retval EFI_SUCCESS Buffer successfully freed.
676 OUT EFI_MEMORY_TYPE
*PoolType OPTIONAL
694 ASSERT(Buffer
!= NULL
);
696 // Get the head & tail of the pool entry
698 Head
= BASE_CR (Buffer
, POOL_HEAD
, Data
);
699 ASSERT(Head
!= NULL
);
701 if (Head
->Signature
!= POOL_HEAD_SIGNATURE
&&
702 Head
->Signature
!= POOLPAGE_HEAD_SIGNATURE
) {
703 ASSERT (Head
->Signature
== POOL_HEAD_SIGNATURE
||
704 Head
->Signature
== POOLPAGE_HEAD_SIGNATURE
);
705 return EFI_INVALID_PARAMETER
;
708 IsGuarded
= IsPoolTypeToGuard (Head
->Type
) &&
709 IsMemoryGuarded ((EFI_PHYSICAL_ADDRESS
)(UINTN
)Head
);
710 HasPoolTail
= !(IsGuarded
&&
711 ((PcdGet8 (PcdHeapGuardPropertyMask
) & BIT7
) == 0));
712 PageAsPool
= (Head
->Signature
== POOLPAGE_HEAD_SIGNATURE
);
715 Tail
= HEAD_TO_TAIL (Head
);
716 ASSERT (Tail
!= NULL
);
721 ASSERT (Tail
->Signature
== POOL_TAIL_SIGNATURE
);
722 ASSERT (Head
->Size
== Tail
->Size
);
724 if (Tail
->Signature
!= POOL_TAIL_SIGNATURE
) {
725 return EFI_INVALID_PARAMETER
;
728 if (Head
->Size
!= Tail
->Size
) {
729 return EFI_INVALID_PARAMETER
;
733 ASSERT_LOCKED (&mPoolMemoryLock
);
736 // Determine the pool type and account for it
739 Pool
= LookupPoolHead (Head
->Type
);
741 return EFI_INVALID_PARAMETER
;
744 DEBUG ((DEBUG_POOL
, "FreePool: %p (len %lx) %,ld\n", Head
->Data
, (UINT64
)(Head
->Size
- POOL_OVERHEAD
), (UINT64
) Pool
->Used
));
746 if (Head
->Type
== EfiACPIReclaimMemory
||
747 Head
->Type
== EfiACPIMemoryNVS
||
748 Head
->Type
== EfiRuntimeServicesCode
||
749 Head
->Type
== EfiRuntimeServicesData
) {
751 Granularity
= RUNTIME_PAGE_ALLOCATION_GRANULARITY
;
753 Granularity
= DEFAULT_PAGE_ALLOCATION_GRANULARITY
;
756 if (PoolType
!= NULL
) {
757 *PoolType
= Head
->Type
;
761 // Determine the pool list
763 Index
= SIZE_TO_LIST(Size
);
764 DEBUG_CLEAR_MEMORY (Head
, Size
);
767 // If it's not on the list, it must be pool pages
769 if (Index
>= SIZE_TO_LIST (Granularity
) || IsGuarded
|| PageAsPool
) {
772 // Return the memory pages back to free memory
774 NoPages
= EFI_SIZE_TO_PAGES (Size
) + EFI_SIZE_TO_PAGES (Granularity
) - 1;
775 NoPages
&= ~(UINTN
)(EFI_SIZE_TO_PAGES (Granularity
) - 1);
777 Head
= AdjustPoolHeadF ((EFI_PHYSICAL_ADDRESS
)(UINTN
)Head
);
778 CoreFreePoolPagesWithGuard (
780 (EFI_PHYSICAL_ADDRESS
)(UINTN
)Head
,
786 (EFI_PHYSICAL_ADDRESS
)(UINTN
)Head
,
794 // Put the pool entry onto the free pool list
796 Free
= (POOL_FREE
*) Head
;
797 ASSERT(Free
!= NULL
);
798 Free
->Signature
= POOL_FREE_SIGNATURE
;
799 Free
->Index
= (UINT32
)Index
;
800 InsertHeadList (&Pool
->FreeList
[Index
], &Free
->Link
);
803 // See if all the pool entries in the same page as Free are freed pool
806 NewPage
= (CHAR8
*)((UINTN
)Free
& ~(Granularity
- 1));
807 Free
= (POOL_FREE
*) &NewPage
[0];
808 ASSERT(Free
!= NULL
);
810 if (Free
->Signature
== POOL_FREE_SIGNATURE
) {
815 while ((Offset
< Granularity
) && (AllFree
)) {
816 Free
= (POOL_FREE
*) &NewPage
[Offset
];
817 ASSERT(Free
!= NULL
);
818 if (Free
->Signature
!= POOL_FREE_SIGNATURE
) {
821 Offset
+= LIST_TO_SIZE(Free
->Index
);
827 // All of the pool entries in the same page as Free are free pool
829 // Remove all of these pool entries from the free loop lists.
831 Free
= (POOL_FREE
*) &NewPage
[0];
832 ASSERT(Free
!= NULL
);
835 while (Offset
< Granularity
) {
836 Free
= (POOL_FREE
*) &NewPage
[Offset
];
837 ASSERT(Free
!= NULL
);
838 RemoveEntryList (&Free
->Link
);
839 Offset
+= LIST_TO_SIZE(Free
->Index
);
845 CoreFreePoolPagesI (Pool
->MemoryType
, (EFI_PHYSICAL_ADDRESS
) (UINTN
)NewPage
,
846 EFI_SIZE_TO_PAGES (Granularity
));
852 // If this is an OS/OEM specific memory type, then check to see if the last
853 // portion of that memory type has been freed. If it has, then free the
854 // list entry for that memory type
856 if (((UINT32
) Pool
->MemoryType
>= MEMORY_TYPE_OEM_RESERVED_MIN
) && Pool
->Used
== 0) {
857 RemoveEntryList (&Pool
->Link
);
858 CoreFreePoolI (Pool
, NULL
);