2 UEFI Heap Guard functions.
4 Copyright (c) 2017-2018, Intel Corporation. All rights reserved.<BR>
5 SPDX-License-Identifier: BSD-2-Clause-Patent
11 #include "HeapGuard.h"
14 // Global to avoid infinite reentrance of memory allocation when updating
15 // page table attributes, which may need allocate pages for new PDE/PTE.
17 GLOBAL_REMOVE_IF_UNREFERENCED BOOLEAN mOnGuarding
= FALSE
;
20 // Pointer to table tracking the Guarded memory with bitmap, in which '1'
21 // is used to indicate memory guarded. '0' might be free memory or Guard
22 // page itself, depending on status of memory adjacent to it.
24 GLOBAL_REMOVE_IF_UNREFERENCED UINT64 mGuardedMemoryMap
= 0;
27 // Current depth level of map table pointed by mGuardedMemoryMap.
28 // mMapLevel must be initialized at least by 1. It will be automatically
29 // updated according to the address of memory just tracked.
31 GLOBAL_REMOVE_IF_UNREFERENCED UINTN mMapLevel
= 1;
34 // Shift and mask for each level of map table
36 GLOBAL_REMOVE_IF_UNREFERENCED UINTN mLevelShift
[GUARDED_HEAP_MAP_TABLE_DEPTH
]
37 = GUARDED_HEAP_MAP_TABLE_DEPTH_SHIFTS
;
38 GLOBAL_REMOVE_IF_UNREFERENCED UINTN mLevelMask
[GUARDED_HEAP_MAP_TABLE_DEPTH
]
39 = GUARDED_HEAP_MAP_TABLE_DEPTH_MASKS
;
42 // Used for promoting freed but not used pages.
44 GLOBAL_REMOVE_IF_UNREFERENCED EFI_PHYSICAL_ADDRESS mLastPromotedPage
= BASE_4GB
;
47 Set corresponding bits in bitmap table to 1 according to the address.
49 @param[in] Address Start address to set for.
50 @param[in] BitNumber Number of bits to set.
51 @param[in] BitMap Pointer to bitmap which covers the Address.
58 IN EFI_PHYSICAL_ADDRESS Address
,
69 StartBit
= (UINTN
)GUARDED_HEAP_MAP_ENTRY_BIT_INDEX (Address
);
70 EndBit
= (StartBit
+ BitNumber
- 1) % GUARDED_HEAP_MAP_ENTRY_BITS
;
72 if ((StartBit
+ BitNumber
) >= GUARDED_HEAP_MAP_ENTRY_BITS
) {
73 Msbs
= (GUARDED_HEAP_MAP_ENTRY_BITS
- StartBit
) %
74 GUARDED_HEAP_MAP_ENTRY_BITS
;
75 Lsbs
= (EndBit
+ 1) % GUARDED_HEAP_MAP_ENTRY_BITS
;
76 Qwords
= (BitNumber
- Msbs
) / GUARDED_HEAP_MAP_ENTRY_BITS
;
84 *BitMap
|= LShiftU64 (LShiftU64 (1, Msbs
) - 1, StartBit
);
91 Qwords
* GUARDED_HEAP_MAP_ENTRY_BYTES
,
98 *BitMap
|= (LShiftU64 (1, Lsbs
) - 1);
103 Set corresponding bits in bitmap table to 0 according to the address.
105 @param[in] Address Start address to set for.
106 @param[in] BitNumber Number of bits to set.
107 @param[in] BitMap Pointer to bitmap which covers the Address.
114 IN EFI_PHYSICAL_ADDRESS Address
,
125 StartBit
= (UINTN
)GUARDED_HEAP_MAP_ENTRY_BIT_INDEX (Address
);
126 EndBit
= (StartBit
+ BitNumber
- 1) % GUARDED_HEAP_MAP_ENTRY_BITS
;
128 if ((StartBit
+ BitNumber
) >= GUARDED_HEAP_MAP_ENTRY_BITS
) {
129 Msbs
= (GUARDED_HEAP_MAP_ENTRY_BITS
- StartBit
) %
130 GUARDED_HEAP_MAP_ENTRY_BITS
;
131 Lsbs
= (EndBit
+ 1) % GUARDED_HEAP_MAP_ENTRY_BITS
;
132 Qwords
= (BitNumber
- Msbs
) / GUARDED_HEAP_MAP_ENTRY_BITS
;
140 *BitMap
&= ~LShiftU64 (LShiftU64 (1, Msbs
) - 1, StartBit
);
145 SetMem64 ((VOID
*)BitMap
, Qwords
* GUARDED_HEAP_MAP_ENTRY_BYTES
, 0);
150 *BitMap
&= ~(LShiftU64 (1, Lsbs
) - 1);
155 Get corresponding bits in bitmap table according to the address.
157 The value of bit 0 corresponds to the status of memory at given Address.
158 No more than 64 bits can be retrieved in one call.
160 @param[in] Address Start address to retrieve bits for.
161 @param[in] BitNumber Number of bits to get.
162 @param[in] BitMap Pointer to bitmap which covers the Address.
164 @return An integer containing the bits information.
169 IN EFI_PHYSICAL_ADDRESS Address
,
180 ASSERT (BitNumber
<= GUARDED_HEAP_MAP_ENTRY_BITS
);
182 StartBit
= (UINTN
)GUARDED_HEAP_MAP_ENTRY_BIT_INDEX (Address
);
183 EndBit
= (StartBit
+ BitNumber
- 1) % GUARDED_HEAP_MAP_ENTRY_BITS
;
185 if ((StartBit
+ BitNumber
) > GUARDED_HEAP_MAP_ENTRY_BITS
) {
186 Msbs
= GUARDED_HEAP_MAP_ENTRY_BITS
- StartBit
;
187 Lsbs
= (EndBit
+ 1) % GUARDED_HEAP_MAP_ENTRY_BITS
;
193 if ((StartBit
== 0) && (BitNumber
== GUARDED_HEAP_MAP_ENTRY_BITS
)) {
196 Result
= RShiftU64 ((*BitMap
), StartBit
) & (LShiftU64 (1, Msbs
) - 1);
199 Result
|= LShiftU64 ((*BitMap
) & (LShiftU64 (1, Lsbs
) - 1), Msbs
);
207 Locate the pointer of bitmap from the guarded memory bitmap tables, which
208 covers the given Address.
210 @param[in] Address Start address to search the bitmap for.
211 @param[in] AllocMapUnit Flag to indicate memory allocation for the table.
212 @param[out] BitMap Pointer to bitmap which covers the Address.
214 @return The bit number from given Address to the end of current map table.
217 FindGuardedMemoryMap (
218 IN EFI_PHYSICAL_ADDRESS Address
,
219 IN BOOLEAN AllocMapUnit
,
234 // Adjust current map table depth according to the address to access
236 while (AllocMapUnit
&&
237 mMapLevel
< GUARDED_HEAP_MAP_TABLE_DEPTH
&&
240 mLevelShift
[GUARDED_HEAP_MAP_TABLE_DEPTH
- mMapLevel
- 1]
243 if (mGuardedMemoryMap
!= 0) {
244 Size
= (mLevelMask
[GUARDED_HEAP_MAP_TABLE_DEPTH
- mMapLevel
- 1] + 1)
245 * GUARDED_HEAP_MAP_ENTRY_BYTES
;
246 Status
= CoreInternalAllocatePages (
249 EFI_SIZE_TO_PAGES (Size
),
253 ASSERT_EFI_ERROR (Status
);
254 ASSERT (MapMemory
!= 0);
256 SetMem ((VOID
*)(UINTN
)MapMemory
, Size
, 0);
258 *(UINT64
*)(UINTN
)MapMemory
= mGuardedMemoryMap
;
259 mGuardedMemoryMap
= MapMemory
;
265 GuardMap
= &mGuardedMemoryMap
;
266 for (Level
= GUARDED_HEAP_MAP_TABLE_DEPTH
- mMapLevel
;
267 Level
< GUARDED_HEAP_MAP_TABLE_DEPTH
;
270 if (*GuardMap
== 0) {
276 Size
= (mLevelMask
[Level
] + 1) * GUARDED_HEAP_MAP_ENTRY_BYTES
;
277 Status
= CoreInternalAllocatePages (
280 EFI_SIZE_TO_PAGES (Size
),
284 ASSERT_EFI_ERROR (Status
);
285 ASSERT (MapMemory
!= 0);
287 SetMem ((VOID
*)(UINTN
)MapMemory
, Size
, 0);
288 *GuardMap
= MapMemory
;
291 Index
= (UINTN
)RShiftU64 (Address
, mLevelShift
[Level
]);
292 Index
&= mLevelMask
[Level
];
293 GuardMap
= (UINT64
*)(UINTN
)((*GuardMap
) + Index
* sizeof (UINT64
));
296 BitsToUnitEnd
= GUARDED_HEAP_MAP_BITS
- GUARDED_HEAP_MAP_BIT_INDEX (Address
);
299 return BitsToUnitEnd
;
303 Set corresponding bits in bitmap table to 1 according to given memory range.
305 @param[in] Address Memory address to guard from.
306 @param[in] NumberOfPages Number of pages to guard.
312 SetGuardedMemoryBits (
313 IN EFI_PHYSICAL_ADDRESS Address
,
314 IN UINTN NumberOfPages
321 while (NumberOfPages
> 0) {
322 BitsToUnitEnd
= FindGuardedMemoryMap (Address
, TRUE
, &BitMap
);
323 ASSERT (BitMap
!= NULL
);
325 if (NumberOfPages
> BitsToUnitEnd
) {
327 Bits
= BitsToUnitEnd
;
329 Bits
= NumberOfPages
;
332 SetBits (Address
, Bits
, BitMap
);
334 NumberOfPages
-= Bits
;
335 Address
+= EFI_PAGES_TO_SIZE (Bits
);
340 Clear corresponding bits in bitmap table according to given memory range.
342 @param[in] Address Memory address to unset from.
343 @param[in] NumberOfPages Number of pages to unset guard.
349 ClearGuardedMemoryBits (
350 IN EFI_PHYSICAL_ADDRESS Address
,
351 IN UINTN NumberOfPages
358 while (NumberOfPages
> 0) {
359 BitsToUnitEnd
= FindGuardedMemoryMap (Address
, TRUE
, &BitMap
);
360 ASSERT (BitMap
!= NULL
);
362 if (NumberOfPages
> BitsToUnitEnd
) {
364 Bits
= BitsToUnitEnd
;
366 Bits
= NumberOfPages
;
369 ClearBits (Address
, Bits
, BitMap
);
371 NumberOfPages
-= Bits
;
372 Address
+= EFI_PAGES_TO_SIZE (Bits
);
377 Retrieve corresponding bits in bitmap table according to given memory range.
379 @param[in] Address Memory address to retrieve from.
380 @param[in] NumberOfPages Number of pages to retrieve.
382 @return An integer containing the guarded memory bitmap.
385 GetGuardedMemoryBits (
386 IN EFI_PHYSICAL_ADDRESS Address
,
387 IN UINTN NumberOfPages
396 ASSERT (NumberOfPages
<= GUARDED_HEAP_MAP_ENTRY_BITS
);
400 while (NumberOfPages
> 0) {
401 BitsToUnitEnd
= FindGuardedMemoryMap (Address
, FALSE
, &BitMap
);
403 if (NumberOfPages
> BitsToUnitEnd
) {
405 Bits
= BitsToUnitEnd
;
407 Bits
= NumberOfPages
;
410 if (BitMap
!= NULL
) {
411 Result
|= LShiftU64 (GetBits (Address
, Bits
, BitMap
), Shift
);
415 NumberOfPages
-= Bits
;
416 Address
+= EFI_PAGES_TO_SIZE (Bits
);
423 Get bit value in bitmap table for the given address.
425 @param[in] Address The address to retrieve for.
432 IN EFI_PHYSICAL_ADDRESS Address
437 FindGuardedMemoryMap (Address
, FALSE
, &GuardMap
);
438 if (GuardMap
!= NULL
) {
441 GUARDED_HEAP_MAP_ENTRY_BIT_INDEX (Address
)
452 Check to see if the page at the given address is a Guard page or not.
454 @param[in] Address The address to check for.
456 @return TRUE The page at Address is a Guard page.
457 @return FALSE The page at Address is not a Guard page.
462 IN EFI_PHYSICAL_ADDRESS Address
468 // There must be at least one guarded page before and/or after given
469 // address if it's a Guard page. The bitmap pattern should be one of
472 BitMap
= GetGuardedMemoryBits (Address
- EFI_PAGE_SIZE
, 3);
473 return ((BitMap
== BIT0
) || (BitMap
== BIT2
) || (BitMap
== (BIT2
| BIT0
)));
477 Check to see if the page at the given address is guarded or not.
479 @param[in] Address The address to check for.
481 @return TRUE The page at Address is guarded.
482 @return FALSE The page at Address is not guarded.
487 IN EFI_PHYSICAL_ADDRESS Address
490 return (GetGuardMapBit (Address
) == 1);
494 Set the page at the given address to be a Guard page.
496 This is done by changing the page table attribute to be NOT PRSENT.
498 @param[in] BaseAddress Page address to Guard at
505 IN EFI_PHYSICAL_ADDRESS BaseAddress
515 // Set flag to make sure allocating memory without GUARD for page table
516 // operation; otherwise infinite loops could be caused.
520 // Note: This might overwrite other attributes needed by other features,
521 // such as NX memory protection.
523 Status
= gCpu
->SetMemoryAttributes (gCpu
, BaseAddress
, EFI_PAGE_SIZE
, EFI_MEMORY_RP
);
524 ASSERT_EFI_ERROR (Status
);
529 Unset the Guard page at the given address to the normal memory.
531 This is done by changing the page table attribute to be PRSENT.
533 @param[in] BaseAddress Page address to Guard at.
540 IN EFI_PHYSICAL_ADDRESS BaseAddress
551 // Once the Guard page is unset, it will be freed back to memory pool. NX
552 // memory protection must be restored for this page if NX is enabled for free
556 if ((PcdGet64 (PcdDxeNxMemoryProtectionPolicy
) & (1 << EfiConventionalMemory
)) != 0) {
557 Attributes
|= EFI_MEMORY_XP
;
561 // Set flag to make sure allocating memory without GUARD for page table
562 // operation; otherwise infinite loops could be caused.
566 // Note: This might overwrite other attributes needed by other features,
567 // such as memory protection (NX). Please make sure they are not enabled
570 Status
= gCpu
->SetMemoryAttributes (gCpu
, BaseAddress
, EFI_PAGE_SIZE
, Attributes
);
571 ASSERT_EFI_ERROR (Status
);
576 Check to see if the memory at the given address should be guarded or not.
578 @param[in] MemoryType Memory type to check.
579 @param[in] AllocateType Allocation type to check.
580 @param[in] PageOrPool Indicate a page allocation or pool allocation.
583 @return TRUE The given type of memory should be guarded.
584 @return FALSE The given type of memory should not be guarded.
587 IsMemoryTypeToGuard (
588 IN EFI_MEMORY_TYPE MemoryType
,
589 IN EFI_ALLOCATE_TYPE AllocateType
,
596 if (AllocateType
== AllocateAddress
) {
600 if ((PcdGet8 (PcdHeapGuardPropertyMask
) & PageOrPool
) == 0) {
604 if (PageOrPool
== GUARD_HEAP_TYPE_POOL
) {
605 ConfigBit
= PcdGet64 (PcdHeapGuardPoolType
);
606 } else if (PageOrPool
== GUARD_HEAP_TYPE_PAGE
) {
607 ConfigBit
= PcdGet64 (PcdHeapGuardPageType
);
609 ConfigBit
= (UINT64
)-1;
612 if ((UINT32
)MemoryType
>= MEMORY_TYPE_OS_RESERVED_MIN
) {
614 } else if ((UINT32
)MemoryType
>= MEMORY_TYPE_OEM_RESERVED_MIN
) {
616 } else if (MemoryType
< EfiMaxMemoryType
) {
617 TestBit
= LShiftU64 (1, MemoryType
);
618 } else if (MemoryType
== EfiMaxMemoryType
) {
619 TestBit
= (UINT64
)-1;
624 return ((ConfigBit
& TestBit
) != 0);
628 Check to see if the pool at the given address should be guarded or not.
630 @param[in] MemoryType Pool type to check.
633 @return TRUE The given type of pool should be guarded.
634 @return FALSE The given type of pool should not be guarded.
638 IN EFI_MEMORY_TYPE MemoryType
641 return IsMemoryTypeToGuard (
649 Check to see if the page at the given address should be guarded or not.
651 @param[in] MemoryType Page type to check.
652 @param[in] AllocateType Allocation type to check.
654 @return TRUE The given type of page should be guarded.
655 @return FALSE The given type of page should not be guarded.
659 IN EFI_MEMORY_TYPE MemoryType
,
660 IN EFI_ALLOCATE_TYPE AllocateType
663 return IsMemoryTypeToGuard (MemoryType
, AllocateType
, GUARD_HEAP_TYPE_PAGE
);
667 Check to see if the heap guard is enabled for page and/or pool allocation.
669 @param[in] GuardType Specify the sub-type(s) of Heap Guard.
678 return IsMemoryTypeToGuard (EfiMaxMemoryType
, AllocateAnyPages
, GuardType
);
682 Set head Guard and tail Guard for the given memory range.
684 @param[in] Memory Base address of memory to set guard for.
685 @param[in] NumberOfPages Memory size in pages.
691 IN EFI_PHYSICAL_ADDRESS Memory
,
692 IN UINTN NumberOfPages
695 EFI_PHYSICAL_ADDRESS GuardPage
;
700 GuardPage
= Memory
+ EFI_PAGES_TO_SIZE (NumberOfPages
);
701 if (!IsGuardPage (GuardPage
)) {
702 SetGuardPage (GuardPage
);
706 GuardPage
= Memory
- EFI_PAGES_TO_SIZE (1);
707 if (!IsGuardPage (GuardPage
)) {
708 SetGuardPage (GuardPage
);
712 // Mark the memory range as Guarded
714 SetGuardedMemoryBits (Memory
, NumberOfPages
);
718 Unset head Guard and tail Guard for the given memory range.
720 @param[in] Memory Base address of memory to unset guard for.
721 @param[in] NumberOfPages Memory size in pages.
726 UnsetGuardForMemory (
727 IN EFI_PHYSICAL_ADDRESS Memory
,
728 IN UINTN NumberOfPages
731 EFI_PHYSICAL_ADDRESS GuardPage
;
734 if (NumberOfPages
== 0) {
739 // Head Guard must be one page before, if any.
742 // -------------------
743 // Head Guard -> 0 1 -> Don't free Head Guard (shared Guard)
744 // Head Guard -> 0 0 -> Free Head Guard either (not shared Guard)
745 // 1 X -> Don't free first page (need a new Guard)
746 // (it'll be turned into a Guard page later)
747 // -------------------
750 GuardPage
= Memory
- EFI_PAGES_TO_SIZE (1);
751 GuardBitmap
= GetGuardedMemoryBits (Memory
- EFI_PAGES_TO_SIZE (2), 2);
752 if ((GuardBitmap
& BIT1
) == 0) {
754 // Head Guard exists.
756 if ((GuardBitmap
& BIT0
) == 0) {
758 // If the head Guard is not a tail Guard of adjacent memory block,
761 UnsetGuardPage (GuardPage
);
765 // Pages before memory to free are still in Guard. It's a partial free
766 // case. Turn first page of memory block to free into a new Guard.
768 SetGuardPage (Memory
);
772 // Tail Guard must be the page after this memory block to free, if any.
775 // --------------------
776 // 1 0 <- Tail Guard -> Don't free Tail Guard (shared Guard)
777 // 0 0 <- Tail Guard -> Free Tail Guard either (not shared Guard)
778 // X 1 -> Don't free last page (need a new Guard)
779 // (it'll be turned into a Guard page later)
780 // --------------------
783 GuardPage
= Memory
+ EFI_PAGES_TO_SIZE (NumberOfPages
);
784 GuardBitmap
= GetGuardedMemoryBits (GuardPage
, 2);
785 if ((GuardBitmap
& BIT0
) == 0) {
787 // Tail Guard exists.
789 if ((GuardBitmap
& BIT1
) == 0) {
791 // If the tail Guard is not a head Guard of adjacent memory block,
792 // free it; otherwise, keep it.
794 UnsetGuardPage (GuardPage
);
798 // Pages after memory to free are still in Guard. It's a partial free
799 // case. We need to keep one page to be a head Guard.
801 SetGuardPage (GuardPage
- EFI_PAGES_TO_SIZE (1));
805 // No matter what, we just clear the mark of the Guarded memory.
807 ClearGuardedMemoryBits (Memory
, NumberOfPages
);
811 Adjust address of free memory according to existing and/or required Guard.
813 This function will check if there're existing Guard pages of adjacent
814 memory blocks, and try to use it as the Guard page of the memory to be
817 @param[in] Start Start address of free memory block.
818 @param[in] Size Size of free memory block.
819 @param[in] SizeRequested Size of memory to allocate.
821 @return The end address of memory block found.
822 @return 0 if no enough space for the required size of memory and its Guard.
828 IN UINT64 SizeRequested
834 // UEFI spec requires that allocated pool must be 8-byte aligned. If it's
835 // indicated to put the pool near the Tail Guard, we need extra bytes to
836 // make sure alignment of the returned pool address.
838 if ((PcdGet8 (PcdHeapGuardPropertyMask
) & BIT7
) == 0) {
839 SizeRequested
= ALIGN_VALUE (SizeRequested
, 8);
842 Target
= Start
+ Size
- SizeRequested
;
843 ASSERT (Target
>= Start
);
848 if (!IsGuardPage (Start
+ Size
)) {
849 // No Guard at tail to share. One more page is needed.
850 Target
-= EFI_PAGES_TO_SIZE (1);
854 if (Target
< Start
) {
859 if (Target
== Start
) {
860 if (!IsGuardPage (Target
- EFI_PAGES_TO_SIZE (1))) {
861 // No enough space for a new head Guard if no Guard at head to share.
866 // OK, we have enough pages for memory and its Guards. Return the End of the
868 return Target
+ SizeRequested
- 1;
872 Adjust the start address and number of pages to free according to Guard.
874 The purpose of this function is to keep the shared Guard page with adjacent
875 memory block if it's still in guard, or free it if no more sharing. Another
876 is to reserve pages as Guard pages in partial page free situation.
878 @param[in,out] Memory Base address of memory to free.
879 @param[in,out] NumberOfPages Size of memory to free.
885 IN OUT EFI_PHYSICAL_ADDRESS
*Memory
,
886 IN OUT UINTN
*NumberOfPages
889 EFI_PHYSICAL_ADDRESS Start
;
890 EFI_PHYSICAL_ADDRESS MemoryToTest
;
894 if ((Memory
== NULL
) || (NumberOfPages
== NULL
) || (*NumberOfPages
== 0)) {
899 PagesToFree
= *NumberOfPages
;
902 // Head Guard must be one page before, if any.
905 // -------------------
906 // Head Guard -> 0 1 -> Don't free Head Guard (shared Guard)
907 // Head Guard -> 0 0 -> Free Head Guard either (not shared Guard)
908 // 1 X -> Don't free first page (need a new Guard)
909 // (it'll be turned into a Guard page later)
910 // -------------------
913 MemoryToTest
= Start
- EFI_PAGES_TO_SIZE (2);
914 GuardBitmap
= GetGuardedMemoryBits (MemoryToTest
, 2);
915 if ((GuardBitmap
& BIT1
) == 0) {
917 // Head Guard exists.
919 if ((GuardBitmap
& BIT0
) == 0) {
921 // If the head Guard is not a tail Guard of adjacent memory block,
922 // free it; otherwise, keep it.
924 Start
-= EFI_PAGES_TO_SIZE (1);
929 // No Head Guard, and pages before memory to free are still in Guard. It's a
930 // partial free case. We need to keep one page to be a tail Guard.
932 Start
+= EFI_PAGES_TO_SIZE (1);
937 // Tail Guard must be the page after this memory block to free, if any.
940 // --------------------
941 // 1 0 <- Tail Guard -> Don't free Tail Guard (shared Guard)
942 // 0 0 <- Tail Guard -> Free Tail Guard either (not shared Guard)
943 // X 1 -> Don't free last page (need a new Guard)
944 // (it'll be turned into a Guard page later)
945 // --------------------
948 MemoryToTest
= Start
+ EFI_PAGES_TO_SIZE (PagesToFree
);
949 GuardBitmap
= GetGuardedMemoryBits (MemoryToTest
, 2);
950 if ((GuardBitmap
& BIT0
) == 0) {
952 // Tail Guard exists.
954 if ((GuardBitmap
& BIT1
) == 0) {
956 // If the tail Guard is not a head Guard of adjacent memory block,
957 // free it; otherwise, keep it.
961 } else if (PagesToFree
> 0) {
963 // No Tail Guard, and pages after memory to free are still in Guard. It's a
964 // partial free case. We need to keep one page to be a head Guard.
970 *NumberOfPages
= PagesToFree
;
974 Adjust the base and number of pages to really allocate according to Guard.
976 @param[in,out] Memory Base address of free memory.
977 @param[in,out] NumberOfPages Size of memory to allocate.
983 IN OUT EFI_PHYSICAL_ADDRESS
*Memory
,
984 IN OUT UINTN
*NumberOfPages
988 // FindFreePages() has already taken the Guard into account. It's safe to
989 // adjust the start address and/or number of pages here, to make sure that
990 // the Guards are also "allocated".
992 if (!IsGuardPage (*Memory
+ EFI_PAGES_TO_SIZE (*NumberOfPages
))) {
993 // No tail Guard, add one.
997 if (!IsGuardPage (*Memory
- EFI_PAGE_SIZE
)) {
998 // No head Guard, add one.
999 *Memory
-= EFI_PAGE_SIZE
;
1000 *NumberOfPages
+= 1;
1005 Adjust the pool head position to make sure the Guard page is adjavent to
1006 pool tail or pool head.
1008 @param[in] Memory Base address of memory allocated.
1009 @param[in] NoPages Number of pages actually allocated.
1010 @param[in] Size Size of memory requested.
1011 (plus pool head/tail overhead)
1013 @return Address of pool head.
1017 IN EFI_PHYSICAL_ADDRESS Memory
,
1022 if ((Memory
== 0) || ((PcdGet8 (PcdHeapGuardPropertyMask
) & BIT7
) != 0)) {
1024 // Pool head is put near the head Guard
1026 return (VOID
*)(UINTN
)Memory
;
1030 // Pool head is put near the tail Guard
1032 Size
= ALIGN_VALUE (Size
, 8);
1033 return (VOID
*)(UINTN
)(Memory
+ EFI_PAGES_TO_SIZE (NoPages
) - Size
);
1037 Get the page base address according to pool head address.
1039 @param[in] Memory Head address of pool to free.
1041 @return Address of pool head.
1045 IN EFI_PHYSICAL_ADDRESS Memory
1048 if ((Memory
== 0) || ((PcdGet8 (PcdHeapGuardPropertyMask
) & BIT7
) != 0)) {
1050 // Pool head is put near the head Guard
1052 return (VOID
*)(UINTN
)Memory
;
1056 // Pool head is put near the tail Guard
1058 return (VOID
*)(UINTN
)(Memory
& ~EFI_PAGE_MASK
);
1062 Allocate or free guarded memory.
1064 @param[in] Start Start address of memory to allocate or free.
1065 @param[in] NumberOfPages Memory size in pages.
1066 @param[in] NewType Memory type to convert to.
1071 CoreConvertPagesWithGuard (
1073 IN UINTN NumberOfPages
,
1074 IN EFI_MEMORY_TYPE NewType
1080 if (NewType
== EfiConventionalMemory
) {
1082 OldPages
= NumberOfPages
;
1084 AdjustMemoryF (&Start
, &NumberOfPages
);
1086 // It's safe to unset Guard page inside memory lock because there should
1087 // be no memory allocation occurred in updating memory page attribute at
1088 // this point. And unsetting Guard page before free will prevent Guard
1089 // page just freed back to pool from being allocated right away before
1090 // marking it usable (from non-present to present).
1092 UnsetGuardForMemory (OldStart
, OldPages
);
1093 if (NumberOfPages
== 0) {
1097 AdjustMemoryA (&Start
, &NumberOfPages
);
1100 return CoreConvertPages (Start
, NumberOfPages
, NewType
);
1104 Set all Guard pages which cannot be set before CPU Arch Protocol installed.
1111 UINTN Entries
[GUARDED_HEAP_MAP_TABLE_DEPTH
];
1112 UINTN Shifts
[GUARDED_HEAP_MAP_TABLE_DEPTH
];
1113 UINTN Indices
[GUARDED_HEAP_MAP_TABLE_DEPTH
];
1114 UINT64 Tables
[GUARDED_HEAP_MAP_TABLE_DEPTH
];
1115 UINT64 Addresses
[GUARDED_HEAP_MAP_TABLE_DEPTH
];
1123 if ((mGuardedMemoryMap
== 0) ||
1125 (mMapLevel
> GUARDED_HEAP_MAP_TABLE_DEPTH
))
1130 CopyMem (Entries
, mLevelMask
, sizeof (Entries
));
1131 CopyMem (Shifts
, mLevelShift
, sizeof (Shifts
));
1133 SetMem (Tables
, sizeof (Tables
), 0);
1134 SetMem (Addresses
, sizeof (Addresses
), 0);
1135 SetMem (Indices
, sizeof (Indices
), 0);
1137 Level
= GUARDED_HEAP_MAP_TABLE_DEPTH
- mMapLevel
;
1138 Tables
[Level
] = mGuardedMemoryMap
;
1143 DumpGuardedMemoryBitmap ();
1147 if (Indices
[Level
] > Entries
[Level
]) {
1151 TableEntry
= ((UINT64
*)(UINTN
)(Tables
[Level
]))[Indices
[Level
]];
1152 Address
= Addresses
[Level
];
1154 if (TableEntry
== 0) {
1156 } else if (Level
< GUARDED_HEAP_MAP_TABLE_DEPTH
- 1) {
1158 Tables
[Level
] = TableEntry
;
1159 Addresses
[Level
] = Address
;
1165 while (Index
< GUARDED_HEAP_MAP_ENTRY_BITS
) {
1166 if ((TableEntry
& 1) == 1) {
1170 GuardPage
= Address
- EFI_PAGE_SIZE
;
1176 GuardPage
= Address
;
1184 if (GuardPage
!= 0) {
1185 SetGuardPage (GuardPage
);
1188 if (TableEntry
== 0) {
1192 TableEntry
= RShiftU64 (TableEntry
, 1);
1193 Address
+= EFI_PAGE_SIZE
;
1199 if (Level
< (GUARDED_HEAP_MAP_TABLE_DEPTH
- (INTN
)mMapLevel
)) {
1203 Indices
[Level
] += 1;
1204 Address
= (Level
== 0) ? 0 : Addresses
[Level
- 1];
1205 Addresses
[Level
] = Address
| LShiftU64 (Indices
[Level
], Shifts
[Level
]);
1210 Find the address of top-most guarded free page.
1212 @param[out] Address Start address of top-most guarded free page.
1217 GetLastGuardedFreePageAddress (
1218 OUT EFI_PHYSICAL_ADDRESS
*Address
1221 EFI_PHYSICAL_ADDRESS AddressGranularity
;
1222 EFI_PHYSICAL_ADDRESS BaseAddress
;
1227 ASSERT (mMapLevel
>= 1);
1230 Map
= mGuardedMemoryMap
;
1231 for (Level
= GUARDED_HEAP_MAP_TABLE_DEPTH
- mMapLevel
;
1232 Level
< GUARDED_HEAP_MAP_TABLE_DEPTH
;
1235 AddressGranularity
= LShiftU64 (1, mLevelShift
[Level
]);
1238 // Find the non-NULL entry at largest index.
1240 for (Index
= (INTN
)mLevelMask
[Level
]; Index
>= 0; --Index
) {
1241 if (((UINT64
*)(UINTN
)Map
)[Index
] != 0) {
1242 BaseAddress
+= MultU64x32 (AddressGranularity
, (UINT32
)Index
);
1243 Map
= ((UINT64
*)(UINTN
)Map
)[Index
];
1250 // Find the non-zero MSB then get the page address.
1253 Map
= RShiftU64 (Map
, 1);
1254 BaseAddress
+= EFI_PAGES_TO_SIZE (1);
1257 *Address
= BaseAddress
;
1263 @param[in] BaseAddress Base address of just freed pages.
1264 @param[in] Pages Number of freed pages.
1270 IN EFI_PHYSICAL_ADDRESS BaseAddress
,
1274 SetGuardedMemoryBits (BaseAddress
, Pages
);
1278 Record freed pages as well as mark them as not-present.
1280 @param[in] BaseAddress Base address of just freed pages.
1281 @param[in] Pages Number of freed pages.
1288 IN EFI_PHYSICAL_ADDRESS BaseAddress
,
1295 // Legacy memory lower than 1MB might be accessed with no allocation. Leave
1298 if (BaseAddress
< BASE_1MB
) {
1302 MarkFreedPages (BaseAddress
, Pages
);
1305 // Set flag to make sure allocating memory without GUARD for page table
1306 // operation; otherwise infinite loops could be caused.
1310 // Note: This might overwrite other attributes needed by other features,
1311 // such as NX memory protection.
1313 Status
= gCpu
->SetMemoryAttributes (
1316 EFI_PAGES_TO_SIZE (Pages
),
1320 // Normally we should ASSERT the returned Status. But there might be memory
1321 // alloc/free involved in SetMemoryAttributes(), which might fail this
1322 // calling. It's rare case so it's OK to let a few tiny holes be not-guarded.
1324 if (EFI_ERROR (Status
)) {
1325 DEBUG ((DEBUG_WARN
, "Failed to guard freed pages: %p (%lu)\n", BaseAddress
, (UINT64
)Pages
));
1328 mOnGuarding
= FALSE
;
1333 Record freed pages as well as mark them as not-present, if enabled.
1335 @param[in] BaseAddress Base address of just freed pages.
1336 @param[in] Pages Number of freed pages.
1342 GuardFreedPagesChecked (
1343 IN EFI_PHYSICAL_ADDRESS BaseAddress
,
1347 if (IsHeapGuardEnabled (GUARD_HEAP_TYPE_FREED
)) {
1348 GuardFreedPages (BaseAddress
, Pages
);
1353 Mark all pages freed before CPU Arch Protocol as not-present.
1357 GuardAllFreedPages (
1361 UINTN Entries
[GUARDED_HEAP_MAP_TABLE_DEPTH
];
1362 UINTN Shifts
[GUARDED_HEAP_MAP_TABLE_DEPTH
];
1363 UINTN Indices
[GUARDED_HEAP_MAP_TABLE_DEPTH
];
1364 UINT64 Tables
[GUARDED_HEAP_MAP_TABLE_DEPTH
];
1365 UINT64 Addresses
[GUARDED_HEAP_MAP_TABLE_DEPTH
];
1371 UINTN GuardPageNumber
;
1373 if ((mGuardedMemoryMap
== 0) ||
1375 (mMapLevel
> GUARDED_HEAP_MAP_TABLE_DEPTH
))
1380 CopyMem (Entries
, mLevelMask
, sizeof (Entries
));
1381 CopyMem (Shifts
, mLevelShift
, sizeof (Shifts
));
1383 SetMem (Tables
, sizeof (Tables
), 0);
1384 SetMem (Addresses
, sizeof (Addresses
), 0);
1385 SetMem (Indices
, sizeof (Indices
), 0);
1387 Level
= GUARDED_HEAP_MAP_TABLE_DEPTH
- mMapLevel
;
1388 Tables
[Level
] = mGuardedMemoryMap
;
1390 GuardPage
= (UINT64
)-1;
1391 GuardPageNumber
= 0;
1394 if (Indices
[Level
] > Entries
[Level
]) {
1398 TableEntry
= ((UINT64
*)(UINTN
)(Tables
[Level
]))[Indices
[Level
]];
1399 Address
= Addresses
[Level
];
1401 if (Level
< GUARDED_HEAP_MAP_TABLE_DEPTH
- 1) {
1403 Tables
[Level
] = TableEntry
;
1404 Addresses
[Level
] = Address
;
1410 while (BitIndex
!= 0) {
1411 if ((TableEntry
& BitIndex
) != 0) {
1412 if (GuardPage
== (UINT64
)-1) {
1413 GuardPage
= Address
;
1417 } else if (GuardPageNumber
> 0) {
1418 GuardFreedPages (GuardPage
, GuardPageNumber
);
1419 GuardPageNumber
= 0;
1420 GuardPage
= (UINT64
)-1;
1423 if (TableEntry
== 0) {
1427 Address
+= EFI_PAGES_TO_SIZE (1);
1428 BitIndex
= LShiftU64 (BitIndex
, 1);
1433 if (Level
< (GUARDED_HEAP_MAP_TABLE_DEPTH
- (INTN
)mMapLevel
)) {
1437 Indices
[Level
] += 1;
1438 Address
= (Level
== 0) ? 0 : Addresses
[Level
- 1];
1439 Addresses
[Level
] = Address
| LShiftU64 (Indices
[Level
], Shifts
[Level
]);
1443 // Update the maximum address of freed page which can be used for memory
1444 // promotion upon out-of-memory-space.
1446 GetLastGuardedFreePageAddress (&Address
);
1448 mLastPromotedPage
= Address
;
1453 This function checks to see if the given memory map descriptor in a memory map
1454 can be merged with any guarded free pages.
1456 @param MemoryMapEntry A pointer to a descriptor in MemoryMap.
1457 @param MaxAddress Maximum address to stop the merge.
1464 IN EFI_MEMORY_DESCRIPTOR
*MemoryMapEntry
,
1465 IN EFI_PHYSICAL_ADDRESS MaxAddress
1468 EFI_PHYSICAL_ADDRESS EndAddress
;
1472 if (!IsHeapGuardEnabled (GUARD_HEAP_TYPE_FREED
) ||
1473 (MemoryMapEntry
->Type
>= EfiMemoryMappedIO
))
1479 Pages
= EFI_SIZE_TO_PAGES ((UINTN
)(MaxAddress
- MemoryMapEntry
->PhysicalStart
));
1480 Pages
-= (INTN
)MemoryMapEntry
->NumberOfPages
;
1483 EndAddress
= MemoryMapEntry
->PhysicalStart
+
1484 EFI_PAGES_TO_SIZE ((UINTN
)MemoryMapEntry
->NumberOfPages
);
1485 Bitmap
= GetGuardedMemoryBits (EndAddress
, GUARDED_HEAP_MAP_ENTRY_BITS
);
1488 if ((Bitmap
& 1) == 0) {
1493 MemoryMapEntry
->NumberOfPages
++;
1494 Bitmap
= RShiftU64 (Bitmap
, 1);
1499 Put part (at most 64 pages a time) guarded free pages back to free page pool.
1501 Freed memory guard is used to detect Use-After-Free (UAF) memory issue, which
1502 makes use of 'Used then throw away' way to detect any illegal access to freed
1503 memory. The thrown-away memory will be marked as not-present so that any access
1504 to those memory (after free) will be caught by page-fault exception.
1506 The problem is that this will consume lots of memory space. Once no memory
1507 left in pool to allocate, we have to restore part of the freed pages to their
1508 normal function. Otherwise the whole system will stop functioning.
1510 @param StartAddress Start address of promoted memory.
1511 @param EndAddress End address of promoted memory.
1513 @return TRUE Succeeded to promote memory.
1514 @return FALSE No free memory found.
1518 PromoteGuardedFreePages (
1519 OUT EFI_PHYSICAL_ADDRESS
*StartAddress
,
1520 OUT EFI_PHYSICAL_ADDRESS
*EndAddress
1524 UINTN AvailablePages
;
1526 EFI_PHYSICAL_ADDRESS Start
;
1528 if (!IsHeapGuardEnabled (GUARD_HEAP_TYPE_FREED
)) {
1533 // Similar to memory allocation service, always search the freed pages in
1534 // descending direction.
1536 Start
= mLastPromotedPage
;
1538 while (AvailablePages
== 0) {
1539 Start
-= EFI_PAGES_TO_SIZE (GUARDED_HEAP_MAP_ENTRY_BITS
);
1541 // If the address wraps around, try the really freed pages at top.
1543 if (Start
> mLastPromotedPage
) {
1544 GetLastGuardedFreePageAddress (&Start
);
1545 ASSERT (Start
!= 0);
1546 Start
-= EFI_PAGES_TO_SIZE (GUARDED_HEAP_MAP_ENTRY_BITS
);
1549 Bitmap
= GetGuardedMemoryBits (Start
, GUARDED_HEAP_MAP_ENTRY_BITS
);
1550 while (Bitmap
> 0) {
1551 if ((Bitmap
& 1) != 0) {
1553 } else if (AvailablePages
== 0) {
1554 Start
+= EFI_PAGES_TO_SIZE (1);
1559 Bitmap
= RShiftU64 (Bitmap
, 1);
1563 if (AvailablePages
!= 0) {
1564 DEBUG ((DEBUG_INFO
, "Promoted pages: %lX (%lx)\r\n", Start
, (UINT64
)AvailablePages
));
1565 ClearGuardedMemoryBits (Start
, AvailablePages
);
1569 // Set flag to make sure allocating memory without GUARD for page table
1570 // operation; otherwise infinite loops could be caused.
1573 Status
= gCpu
->SetMemoryAttributes (gCpu
, Start
, EFI_PAGES_TO_SIZE (AvailablePages
), 0);
1574 ASSERT_EFI_ERROR (Status
);
1575 mOnGuarding
= FALSE
;
1578 mLastPromotedPage
= Start
;
1579 *StartAddress
= Start
;
1580 *EndAddress
= Start
+ EFI_PAGES_TO_SIZE (AvailablePages
) - 1;
1588 Notify function used to set all Guard pages before CPU Arch Protocol installed.
1591 HeapGuardCpuArchProtocolNotify (
1595 ASSERT (gCpu
!= NULL
);
1597 if (IsHeapGuardEnabled (GUARD_HEAP_TYPE_PAGE
|GUARD_HEAP_TYPE_POOL
) &&
1598 IsHeapGuardEnabled (GUARD_HEAP_TYPE_FREED
))
1600 DEBUG ((DEBUG_ERROR
, "Heap guard and freed memory guard cannot be enabled at the same time.\n"));
1604 if (IsHeapGuardEnabled (GUARD_HEAP_TYPE_PAGE
|GUARD_HEAP_TYPE_POOL
)) {
1605 SetAllGuardPages ();
1608 if (IsHeapGuardEnabled (GUARD_HEAP_TYPE_FREED
)) {
1609 GuardAllFreedPages ();
1614 Helper function to convert a UINT64 value in binary to a string.
1616 @param[in] Value Value of a UINT64 integer.
1617 @param[out] BinString String buffer to contain the conversion result.
1624 OUT CHAR8
*BinString
1629 if (BinString
== NULL
) {
1633 for (Index
= 64; Index
> 0; --Index
) {
1634 BinString
[Index
- 1] = '0' + (Value
& 1);
1635 Value
= RShiftU64 (Value
, 1);
1638 BinString
[64] = '\0';
1642 Dump the guarded memory bit map.
1646 DumpGuardedMemoryBitmap (
1650 UINTN Entries
[GUARDED_HEAP_MAP_TABLE_DEPTH
];
1651 UINTN Shifts
[GUARDED_HEAP_MAP_TABLE_DEPTH
];
1652 UINTN Indices
[GUARDED_HEAP_MAP_TABLE_DEPTH
];
1653 UINT64 Tables
[GUARDED_HEAP_MAP_TABLE_DEPTH
];
1654 UINT64 Addresses
[GUARDED_HEAP_MAP_TABLE_DEPTH
];
1659 CHAR8 String
[GUARDED_HEAP_MAP_ENTRY_BITS
+ 1];
1663 if (!IsHeapGuardEnabled (GUARD_HEAP_TYPE_ALL
)) {
1667 if ((mGuardedMemoryMap
== 0) ||
1669 (mMapLevel
> GUARDED_HEAP_MAP_TABLE_DEPTH
))
1674 Ruler1
= " 3 2 1 0";
1675 Ruler2
= "FEDCBA9876543210FEDCBA9876543210FEDCBA9876543210FEDCBA9876543210";
1678 HEAP_GUARD_DEBUG_LEVEL
,
1679 "============================="
1680 " Guarded Memory Bitmap "
1681 "==============================\r\n"
1683 DEBUG ((HEAP_GUARD_DEBUG_LEVEL
, " %a\r\n", Ruler1
));
1684 DEBUG ((HEAP_GUARD_DEBUG_LEVEL
, " %a\r\n", Ruler2
));
1686 CopyMem (Entries
, mLevelMask
, sizeof (Entries
));
1687 CopyMem (Shifts
, mLevelShift
, sizeof (Shifts
));
1689 SetMem (Indices
, sizeof (Indices
), 0);
1690 SetMem (Tables
, sizeof (Tables
), 0);
1691 SetMem (Addresses
, sizeof (Addresses
), 0);
1693 Level
= GUARDED_HEAP_MAP_TABLE_DEPTH
- mMapLevel
;
1694 Tables
[Level
] = mGuardedMemoryMap
;
1699 if (Indices
[Level
] > Entries
[Level
]) {
1705 HEAP_GUARD_DEBUG_LEVEL
,
1706 "========================================="
1707 "=========================================\r\n"
1710 TableEntry
= ((UINT64
*)(UINTN
)Tables
[Level
])[Indices
[Level
]];
1711 Address
= Addresses
[Level
];
1713 if (TableEntry
== 0) {
1714 if (Level
== GUARDED_HEAP_MAP_TABLE_DEPTH
- 1) {
1715 if (RepeatZero
== 0) {
1716 Uint64ToBinString (TableEntry
, String
);
1717 DEBUG ((HEAP_GUARD_DEBUG_LEVEL
, "%016lx: %a\r\n", Address
, String
));
1718 } else if (RepeatZero
== 1) {
1719 DEBUG ((HEAP_GUARD_DEBUG_LEVEL
, "... : ...\r\n"));
1724 } else if (Level
< GUARDED_HEAP_MAP_TABLE_DEPTH
- 1) {
1726 Tables
[Level
] = TableEntry
;
1727 Addresses
[Level
] = Address
;
1734 Uint64ToBinString (TableEntry
, String
);
1735 DEBUG ((HEAP_GUARD_DEBUG_LEVEL
, "%016lx: %a\r\n", Address
, String
));
1739 if (Level
< (GUARDED_HEAP_MAP_TABLE_DEPTH
- (INTN
)mMapLevel
)) {
1743 Indices
[Level
] += 1;
1744 Address
= (Level
== 0) ? 0 : Addresses
[Level
- 1];
1745 Addresses
[Level
] = Address
| LShiftU64 (Indices
[Level
], Shifts
[Level
]);