2 UEFI Memory pool management functions.
4 Copyright (c) 2006 - 2018, Intel Corporation. All rights reserved.<BR>
5 SPDX-License-Identifier: BSD-2-Clause-Patent
11 #include "HeapGuard.h"
13 STATIC EFI_LOCK mPoolMemoryLock
= EFI_INITIALIZE_LOCK_VARIABLE (TPL_NOTIFY
);
15 #define POOL_FREE_SIGNATURE SIGNATURE_32('p','f','r','0')
22 #define POOL_HEAD_SIGNATURE SIGNATURE_32('p','h','d','0')
23 #define POOLPAGE_HEAD_SIGNATURE SIGNATURE_32('p','h','d','1')
32 #define SIZE_OF_POOL_HEAD OFFSET_OF(POOL_HEAD,Data)
34 #define POOL_TAIL_SIGNATURE SIGNATURE_32('p','t','a','l')
41 #define POOL_OVERHEAD (SIZE_OF_POOL_HEAD + sizeof(POOL_TAIL))
43 #define HEAD_TO_TAIL(a) \
44 ((POOL_TAIL *) (((CHAR8 *) (a)) + (a)->Size - sizeof(POOL_TAIL)));
47 // Each element is the sum of the 2 previous ones: this allows us to migrate
48 // blocks between bins by splitting them up, while not wasting too much memory
49 // as we would in a strict power-of-2 sequence
51 STATIC CONST UINT16 mPoolSizeTable
[] = {
52 128, 256, 384, 640, 1024, 1664, 2688, 4352, 7040, 11392, 18432, 29824
55 #define SIZE_TO_LIST(a) (GetPoolIndexFromSize (a))
56 #define LIST_TO_SIZE(a) (mPoolSizeTable [a])
58 #define MAX_POOL_LIST (ARRAY_SIZE (mPoolSizeTable))
60 #define MAX_POOL_SIZE (MAX_ADDRESS - POOL_OVERHEAD)
66 #define POOL_SIGNATURE SIGNATURE_32('p','l','s','t')
70 EFI_MEMORY_TYPE MemoryType
;
71 LIST_ENTRY FreeList
[MAX_POOL_LIST
];
76 // Pool header for each memory type.
78 POOL mPoolHead
[EfiMaxMemoryType
];
81 // List of pool header to search for the appropriate memory type.
83 LIST_ENTRY mPoolHeadList
= INITIALIZE_LIST_HEAD_VARIABLE (mPoolHeadList
);
86 Get pool size table index from the specified size.
88 @param Size The specified size to get index from pool table.
90 @return The index of pool size table.
95 GetPoolIndexFromSize (
101 for (Index
= 0; Index
< MAX_POOL_LIST
; Index
++) {
102 if (mPoolSizeTable
[Index
] >= Size
) {
107 return MAX_POOL_LIST
;
111 Called to initialize the pool.
122 for (Type
= 0; Type
< EfiMaxMemoryType
; Type
++) {
123 mPoolHead
[Type
].Signature
= 0;
124 mPoolHead
[Type
].Used
= 0;
125 mPoolHead
[Type
].MemoryType
= (EFI_MEMORY_TYPE
)Type
;
126 for (Index
= 0; Index
< MAX_POOL_LIST
; Index
++) {
127 InitializeListHead (&mPoolHead
[Type
].FreeList
[Index
]);
133 Look up pool head for specified memory type.
135 @param MemoryType Memory type of which pool head is looked for
137 @return Pointer of Corresponding pool head.
142 IN EFI_MEMORY_TYPE MemoryType
149 if ((UINT32
)MemoryType
< EfiMaxMemoryType
) {
150 return &mPoolHead
[MemoryType
];
154 // MemoryType values in the range 0x80000000..0xFFFFFFFF are reserved for use by UEFI
155 // OS loaders that are provided by operating system vendors.
156 // MemoryType values in the range 0x70000000..0x7FFFFFFF are reserved for OEM use.
158 if ((UINT32
)MemoryType
>= MEMORY_TYPE_OEM_RESERVED_MIN
) {
159 for (Link
= mPoolHeadList
.ForwardLink
; Link
!= &mPoolHeadList
; Link
= Link
->ForwardLink
) {
160 Pool
= CR (Link
, POOL
, Link
, POOL_SIGNATURE
);
161 if (Pool
->MemoryType
== MemoryType
) {
166 Pool
= CoreAllocatePoolI (EfiBootServicesData
, sizeof (POOL
), FALSE
);
171 Pool
->Signature
= POOL_SIGNATURE
;
173 Pool
->MemoryType
= MemoryType
;
174 for (Index
= 0; Index
< MAX_POOL_LIST
; Index
++) {
175 InitializeListHead (&Pool
->FreeList
[Index
]);
178 InsertHeadList (&mPoolHeadList
, &Pool
->Link
);
187 Allocate pool of a particular type.
189 @param PoolType Type of pool to allocate
190 @param Size The amount of pool to allocate
191 @param Buffer The address to return a pointer to the allocated
194 @retval EFI_INVALID_PARAMETER Buffer is NULL.
195 PoolType is in the range EfiMaxMemoryType..0x6FFFFFFF.
196 PoolType is EfiPersistentMemory.
197 @retval EFI_OUT_OF_RESOURCES Size exceeds max pool size or allocation failed.
198 @retval EFI_SUCCESS Pool successfully allocated.
203 CoreInternalAllocatePool (
204 IN EFI_MEMORY_TYPE PoolType
,
213 // If it's not a valid type, fail it
215 if (((PoolType
>= EfiMaxMemoryType
) && (PoolType
< MEMORY_TYPE_OEM_RESERVED_MIN
)) ||
216 (PoolType
== EfiConventionalMemory
) || (PoolType
== EfiPersistentMemory
) || (PoolType
== EfiUnacceptedMemoryType
))
218 return EFI_INVALID_PARAMETER
;
221 if (Buffer
== NULL
) {
222 return EFI_INVALID_PARAMETER
;
228 // If size is too large, fail it
229 // Base on the EFI spec, return status of EFI_OUT_OF_RESOURCES
231 if (Size
> MAX_POOL_SIZE
) {
232 return EFI_OUT_OF_RESOURCES
;
235 NeedGuard
= IsPoolTypeToGuard (PoolType
) && !mOnGuarding
;
238 // Acquire the memory lock and make the allocation
240 Status
= CoreAcquireLockOrFail (&mPoolMemoryLock
);
241 if (EFI_ERROR (Status
)) {
242 return EFI_OUT_OF_RESOURCES
;
245 *Buffer
= CoreAllocatePoolI (PoolType
, Size
, NeedGuard
);
246 CoreReleaseLock (&mPoolMemoryLock
);
247 return (*Buffer
!= NULL
) ? EFI_SUCCESS
: EFI_OUT_OF_RESOURCES
;
251 Allocate pool of a particular type.
253 @param PoolType Type of pool to allocate
254 @param Size The amount of pool to allocate
255 @param Buffer The address to return a pointer to the allocated
258 @retval EFI_INVALID_PARAMETER Buffer is NULL.
259 PoolType is in the range EfiMaxMemoryType..0x6FFFFFFF.
260 PoolType is EfiPersistentMemory.
261 @retval EFI_OUT_OF_RESOURCES Size exceeds max pool size or allocation failed.
262 @retval EFI_SUCCESS Pool successfully allocated.
268 IN EFI_MEMORY_TYPE PoolType
,
275 Status
= CoreInternalAllocatePool (PoolType
, Size
, Buffer
);
276 if (!EFI_ERROR (Status
)) {
278 (EFI_PHYSICAL_ADDRESS
)(UINTN
)RETURN_ADDRESS (0),
279 MemoryProfileActionAllocatePool
,
285 InstallMemoryAttributesTableOnMemoryAllocation (PoolType
);
292 Internal function. Used by the pool functions to allocate pages
293 to back pool allocation requests.
295 @param PoolType The type of memory for the new pool pages
296 @param NoPages No of pages to allocate
297 @param Granularity Bits to align.
298 @param NeedGuard Flag to indicate Guard page is needed or not
300 @return The allocated memory, or NULL
305 CoreAllocatePoolPagesI (
306 IN EFI_MEMORY_TYPE PoolType
,
308 IN UINTN Granularity
,
315 Status
= CoreAcquireLockOrFail (&gMemoryLock
);
316 if (EFI_ERROR (Status
)) {
320 Buffer
= CoreAllocatePoolPages (PoolType
, NoPages
, Granularity
, NeedGuard
);
321 CoreReleaseMemoryLock ();
323 if (Buffer
!= NULL
) {
325 SetGuardForMemory ((EFI_PHYSICAL_ADDRESS
)(UINTN
)Buffer
, NoPages
);
328 ApplyMemoryProtectionPolicy (
329 EfiConventionalMemory
,
331 (EFI_PHYSICAL_ADDRESS
)(UINTN
)Buffer
,
332 EFI_PAGES_TO_SIZE (NoPages
)
340 Internal function to allocate pool of a particular type.
341 Caller must have the memory lock held
343 @param PoolType Type of pool to allocate
344 @param Size The amount of pool to allocate
345 @param NeedGuard Flag to indicate Guard page is needed or not
347 @return The allocate pool, or NULL
352 IN EFI_MEMORY_TYPE PoolType
,
365 UINTN Offset
, MaxOffset
;
371 ASSERT_LOCKED (&mPoolMemoryLock
);
373 if ((PoolType
== EfiACPIReclaimMemory
) ||
374 (PoolType
== EfiACPIMemoryNVS
) ||
375 (PoolType
== EfiRuntimeServicesCode
) ||
376 (PoolType
== EfiRuntimeServicesData
))
378 Granularity
= RUNTIME_PAGE_ALLOCATION_GRANULARITY
;
380 Granularity
= DEFAULT_PAGE_ALLOCATION_GRANULARITY
;
384 // Adjust the size by the pool header & tail overhead
387 HasPoolTail
= !(NeedGuard
&&
388 ((PcdGet8 (PcdHeapGuardPropertyMask
) & BIT7
) == 0));
389 PageAsPool
= (IsHeapGuardEnabled (GUARD_HEAP_TYPE_FREED
) && !mOnGuarding
);
392 // Adjusting the Size to be of proper alignment so that
393 // we don't get an unaligned access fault later when
394 // pool_Tail is being initialized
396 Size
= ALIGN_VARIABLE (Size
);
398 Size
+= POOL_OVERHEAD
;
399 Index
= SIZE_TO_LIST (Size
);
400 Pool
= LookupPoolHead (PoolType
);
408 // If allocation is over max size, just allocate pages for the request
411 if ((Index
>= SIZE_TO_LIST (Granularity
)) || NeedGuard
|| PageAsPool
) {
413 Size
-= sizeof (POOL_TAIL
);
416 NoPages
= EFI_SIZE_TO_PAGES (Size
) + EFI_SIZE_TO_PAGES (Granularity
) - 1;
417 NoPages
&= ~(UINTN
)(EFI_SIZE_TO_PAGES (Granularity
) - 1);
418 Head
= CoreAllocatePoolPagesI (PoolType
, NoPages
, Granularity
, NeedGuard
);
420 Head
= AdjustPoolHeadA ((EFI_PHYSICAL_ADDRESS
)(UINTN
)Head
, NoPages
, Size
);
427 // If there's no free pool in the proper list size, go get some more pages
429 if (IsListEmpty (&Pool
->FreeList
[Index
])) {
430 Offset
= LIST_TO_SIZE (Index
);
431 MaxOffset
= Granularity
;
434 // Check the bins holding larger blocks, and carve one up if needed
436 while (++Index
< SIZE_TO_LIST (Granularity
)) {
437 if (!IsListEmpty (&Pool
->FreeList
[Index
])) {
438 Free
= CR (Pool
->FreeList
[Index
].ForwardLink
, POOL_FREE
, Link
, POOL_FREE_SIGNATURE
);
439 RemoveEntryList (&Free
->Link
);
440 NewPage
= (VOID
*)Free
;
441 MaxOffset
= LIST_TO_SIZE (Index
);
449 NewPage
= CoreAllocatePoolPagesI (
451 EFI_SIZE_TO_PAGES (Granularity
),
455 if (NewPage
== NULL
) {
460 // Serve the allocation request from the head of the allocated block
463 Head
= (POOL_HEAD
*)NewPage
;
466 // Carve up remaining space into free pool blocks
469 while (Offset
< MaxOffset
) {
470 ASSERT (Index
< MAX_POOL_LIST
);
471 FSize
= LIST_TO_SIZE (Index
);
473 while (Offset
+ FSize
<= MaxOffset
) {
474 Free
= (POOL_FREE
*)&NewPage
[Offset
];
475 Free
->Signature
= POOL_FREE_SIGNATURE
;
476 Free
->Index
= (UINT32
)Index
;
477 InsertHeadList (&Pool
->FreeList
[Index
], &Free
->Link
);
484 ASSERT (Offset
== MaxOffset
);
489 // Remove entry from free pool list
491 Free
= CR (Pool
->FreeList
[Index
].ForwardLink
, POOL_FREE
, Link
, POOL_FREE_SIGNATURE
);
492 RemoveEntryList (&Free
->Link
);
494 Head
= (POOL_HEAD
*)Free
;
501 // Account the allocation
506 // If we have a pool buffer, fill in the header & tail info
508 Head
->Signature
= (PageAsPool
) ? POOLPAGE_HEAD_SIGNATURE
: POOL_HEAD_SIGNATURE
;
510 Head
->Type
= (EFI_MEMORY_TYPE
)PoolType
;
514 Tail
= HEAD_TO_TAIL (Head
);
515 Tail
->Signature
= POOL_TAIL_SIGNATURE
;
518 Size
-= POOL_OVERHEAD
;
520 Size
-= SIZE_OF_POOL_HEAD
;
523 DEBUG_CLEAR_MEMORY (Buffer
, Size
);
527 "AllocatePoolI: Type %x, Addr %p (len %lx) %,ld\n",
534 DEBUG ((DEBUG_ERROR
| DEBUG_POOL
, "AllocatePool: failed to allocate %ld bytes\n", (UINT64
)Size
));
543 @param Buffer The allocated pool entry to free
544 @param PoolType Pointer to pool type
546 @retval EFI_INVALID_PARAMETER Buffer is not a valid value.
547 @retval EFI_SUCCESS Pool successfully freed.
552 CoreInternalFreePool (
554 OUT EFI_MEMORY_TYPE
*PoolType OPTIONAL
559 if (Buffer
== NULL
) {
560 return EFI_INVALID_PARAMETER
;
563 CoreAcquireLock (&mPoolMemoryLock
);
564 Status
= CoreFreePoolI (Buffer
, PoolType
);
565 CoreReleaseLock (&mPoolMemoryLock
);
572 @param Buffer The allocated pool entry to free
574 @retval EFI_INVALID_PARAMETER Buffer is not a valid value.
575 @retval EFI_SUCCESS Pool successfully freed.
585 EFI_MEMORY_TYPE PoolType
;
587 Status
= CoreInternalFreePool (Buffer
, &PoolType
);
588 if (!EFI_ERROR (Status
)) {
590 (EFI_PHYSICAL_ADDRESS
)(UINTN
)RETURN_ADDRESS (0),
591 MemoryProfileActionFreePool
,
597 InstallMemoryAttributesTableOnMemoryAllocation (PoolType
);
604 Internal function. Frees pool pages allocated via CoreAllocatePoolPagesI().
606 @param PoolType The type of memory for the pool pages
607 @param Memory The base address to free
608 @param NoPages The number of pages to free
614 IN EFI_MEMORY_TYPE PoolType
,
615 IN EFI_PHYSICAL_ADDRESS Memory
,
619 CoreAcquireMemoryLock ();
620 CoreFreePoolPages (Memory
, NoPages
);
621 CoreReleaseMemoryLock ();
623 GuardFreedPagesChecked (Memory
, NoPages
);
624 ApplyMemoryProtectionPolicy (
626 EfiConventionalMemory
,
627 (EFI_PHYSICAL_ADDRESS
)(UINTN
)Memory
,
628 EFI_PAGES_TO_SIZE (NoPages
)
633 Internal function. Frees guarded pool pages.
635 @param PoolType The type of memory for the pool pages
636 @param Memory The base address to free
637 @param NoPages The number of pages to free
642 CoreFreePoolPagesWithGuard (
643 IN EFI_MEMORY_TYPE PoolType
,
644 IN EFI_PHYSICAL_ADDRESS Memory
,
648 EFI_PHYSICAL_ADDRESS MemoryGuarded
;
649 UINTN NoPagesGuarded
;
651 MemoryGuarded
= Memory
;
652 NoPagesGuarded
= NoPages
;
654 AdjustMemoryF (&Memory
, &NoPages
);
656 // It's safe to unset Guard page inside memory lock because there should
657 // be no memory allocation occurred in updating memory page attribute at
658 // this point. And unsetting Guard page before free will prevent Guard
659 // page just freed back to pool from being allocated right away before
660 // marking it usable (from non-present to present).
662 UnsetGuardForMemory (MemoryGuarded
, NoPagesGuarded
);
664 CoreFreePoolPagesI (PoolType
, Memory
, NoPages
);
669 Internal function to free a pool entry.
670 Caller must have the memory lock held
672 @param Buffer The allocated pool entry to free
673 @param PoolType Pointer to pool type
675 @retval EFI_INVALID_PARAMETER Buffer not valid
676 @retval EFI_SUCCESS Buffer successfully freed.
682 OUT EFI_MEMORY_TYPE
*PoolType OPTIONAL
700 ASSERT (Buffer
!= NULL
);
702 // Get the head & tail of the pool entry
704 Head
= BASE_CR (Buffer
, POOL_HEAD
, Data
);
705 ASSERT (Head
!= NULL
);
707 if ((Head
->Signature
!= POOL_HEAD_SIGNATURE
) &&
708 (Head
->Signature
!= POOLPAGE_HEAD_SIGNATURE
))
711 Head
->Signature
== POOL_HEAD_SIGNATURE
||
712 Head
->Signature
== POOLPAGE_HEAD_SIGNATURE
714 return EFI_INVALID_PARAMETER
;
717 IsGuarded
= IsPoolTypeToGuard (Head
->Type
) &&
718 IsMemoryGuarded ((EFI_PHYSICAL_ADDRESS
)(UINTN
)Head
);
719 HasPoolTail
= !(IsGuarded
&&
720 ((PcdGet8 (PcdHeapGuardPropertyMask
) & BIT7
) == 0));
721 PageAsPool
= (Head
->Signature
== POOLPAGE_HEAD_SIGNATURE
);
724 Tail
= HEAD_TO_TAIL (Head
);
725 ASSERT (Tail
!= NULL
);
730 ASSERT (Tail
->Signature
== POOL_TAIL_SIGNATURE
);
731 ASSERT (Head
->Size
== Tail
->Size
);
733 if (Tail
->Signature
!= POOL_TAIL_SIGNATURE
) {
734 return EFI_INVALID_PARAMETER
;
737 if (Head
->Size
!= Tail
->Size
) {
738 return EFI_INVALID_PARAMETER
;
742 ASSERT_LOCKED (&mPoolMemoryLock
);
745 // Determine the pool type and account for it
748 Pool
= LookupPoolHead (Head
->Type
);
750 return EFI_INVALID_PARAMETER
;
754 DEBUG ((DEBUG_POOL
, "FreePool: %p (len %lx) %,ld\n", Head
->Data
, (UINT64
)(Head
->Size
- POOL_OVERHEAD
), (UINT64
)Pool
->Used
));
756 if ((Head
->Type
== EfiACPIReclaimMemory
) ||
757 (Head
->Type
== EfiACPIMemoryNVS
) ||
758 (Head
->Type
== EfiRuntimeServicesCode
) ||
759 (Head
->Type
== EfiRuntimeServicesData
))
761 Granularity
= RUNTIME_PAGE_ALLOCATION_GRANULARITY
;
763 Granularity
= DEFAULT_PAGE_ALLOCATION_GRANULARITY
;
766 if (PoolType
!= NULL
) {
767 *PoolType
= Head
->Type
;
771 // Determine the pool list
773 Index
= SIZE_TO_LIST (Size
);
774 DEBUG_CLEAR_MEMORY (Head
, Size
);
777 // If it's not on the list, it must be pool pages
779 if ((Index
>= SIZE_TO_LIST (Granularity
)) || IsGuarded
|| PageAsPool
) {
781 // Return the memory pages back to free memory
783 NoPages
= EFI_SIZE_TO_PAGES (Size
) + EFI_SIZE_TO_PAGES (Granularity
) - 1;
784 NoPages
&= ~(UINTN
)(EFI_SIZE_TO_PAGES (Granularity
) - 1);
786 Head
= AdjustPoolHeadF ((EFI_PHYSICAL_ADDRESS
)(UINTN
)Head
);
787 CoreFreePoolPagesWithGuard (
789 (EFI_PHYSICAL_ADDRESS
)(UINTN
)Head
,
795 (EFI_PHYSICAL_ADDRESS
)(UINTN
)Head
,
801 // Put the pool entry onto the free pool list
803 Free
= (POOL_FREE
*)Head
;
804 ASSERT (Free
!= NULL
);
805 Free
->Signature
= POOL_FREE_SIGNATURE
;
806 Free
->Index
= (UINT32
)Index
;
807 InsertHeadList (&Pool
->FreeList
[Index
], &Free
->Link
);
810 // See if all the pool entries in the same page as Free are freed pool
813 NewPage
= (CHAR8
*)((UINTN
)Free
& ~(Granularity
- 1));
814 Free
= (POOL_FREE
*)&NewPage
[0];
815 ASSERT (Free
!= NULL
);
817 if (Free
->Signature
== POOL_FREE_SIGNATURE
) {
821 while ((Offset
< Granularity
) && (AllFree
)) {
822 Free
= (POOL_FREE
*)&NewPage
[Offset
];
823 ASSERT (Free
!= NULL
);
824 if (Free
->Signature
!= POOL_FREE_SIGNATURE
) {
828 Offset
+= LIST_TO_SIZE (Free
->Index
);
833 // All of the pool entries in the same page as Free are free pool
835 // Remove all of these pool entries from the free loop lists.
837 Free
= (POOL_FREE
*)&NewPage
[0];
838 ASSERT (Free
!= NULL
);
841 while (Offset
< Granularity
) {
842 Free
= (POOL_FREE
*)&NewPage
[Offset
];
843 ASSERT (Free
!= NULL
);
844 RemoveEntryList (&Free
->Link
);
845 Offset
+= LIST_TO_SIZE (Free
->Index
);
853 (EFI_PHYSICAL_ADDRESS
)(UINTN
)NewPage
,
854 EFI_SIZE_TO_PAGES (Granularity
)
861 // If this is an OS/OEM specific memory type, then check to see if the last
862 // portion of that memory type has been freed. If it has, then free the
863 // list entry for that memory type
865 if (((UINT32
)Pool
->MemoryType
>= MEMORY_TYPE_OEM_RESERVED_MIN
) && (Pool
->Used
== 0)) {
866 RemoveEntryList (&Pool
->Link
);
867 CoreFreePoolI (Pool
, NULL
);