2 x64 Virtual Memory Management Services in the form of an IA-32 driver.
3 Used to establish a 1:1 Virtual to Physical Mapping that is required to
4 enter Long Mode (x64 64-bit mode).
6 While we make a 1:1 mapping (identity mapping) for all physical pages
7 we still need to use the MTRR's to ensure that the cachability attributes
8 for all memory regions is correct.
10 The basic idea is to use 2MB page table entries where ever possible. If
11 more granularity of cachability is required then 4K page tables are used.
14 1) IA-32 Intel(R) Architecture Software Developer's Manual Volume 1:Basic Architecture, Intel
15 2) IA-32 Intel(R) Architecture Software Developer's Manual Volume 2:Instruction Set Reference, Intel
16 3) IA-32 Intel(R) Architecture Software Developer's Manual Volume 3:System Programmer's Guide, Intel
18 Copyright (c) 2006 - 2022, Intel Corporation. All rights reserved.<BR>
19 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
21 SPDX-License-Identifier: BSD-2-Clause-Patent
25 #include <Register/Intel/Cpuid.h>
27 #include "VirtualMemory.h"
30 // Global variable to keep track current available memory used as page table.
32 PAGE_TABLE_POOL
*mPageTablePool
= NULL
;
35 Clear legacy memory located at the first 4K-page, if available.
37 This function traverses the whole HOB list to check if memory from 0 to 4095
38 exists and has not been allocated, and then clear it if so.
40 @param HobStart The start of HobList passed to DxeCore.
48 EFI_PEI_HOB_POINTERS RscHob
;
49 EFI_PEI_HOB_POINTERS MemHob
;
52 RscHob
.Raw
= HobStart
;
53 MemHob
.Raw
= HobStart
;
57 // Check if page 0 exists and free
59 while ((RscHob
.Raw
= GetNextHob (
60 EFI_HOB_TYPE_RESOURCE_DESCRIPTOR
,
64 if ((RscHob
.ResourceDescriptor
->ResourceType
== EFI_RESOURCE_SYSTEM_MEMORY
) &&
65 (RscHob
.ResourceDescriptor
->PhysicalStart
== 0))
69 // Make sure memory at 0-4095 has not been allocated.
71 while ((MemHob
.Raw
= GetNextHob (
72 EFI_HOB_TYPE_MEMORY_ALLOCATION
,
76 if (MemHob
.MemoryAllocation
->AllocDescriptor
.MemoryBaseAddress
83 MemHob
.Raw
= GET_NEXT_HOB (MemHob
);
89 RscHob
.Raw
= GET_NEXT_HOB (RscHob
);
93 DEBUG ((DEBUG_INFO
, "Clearing first 4K-page!\r\n"));
94 SetMem (NULL
, EFI_PAGE_SIZE
, 0);
101 Return configure status of NULL pointer detection feature.
103 @return TRUE NULL pointer detection feature is enabled
104 @return FALSE NULL pointer detection feature is disabled
108 IsNullDetectionEnabled (
112 return ((PcdGet8 (PcdNullPointerDetectionPropertyMask
) & BIT0
) != 0);
116 The function will check if Execute Disable Bit is available.
118 @retval TRUE Execute Disable Bit is available.
119 @retval FALSE Execute Disable Bit is not available.
123 IsExecuteDisableBitAvailable (
132 AsmCpuid (0x80000000, &RegEax
, NULL
, NULL
, NULL
);
133 if (RegEax
>= 0x80000001) {
134 AsmCpuid (0x80000001, NULL
, NULL
, NULL
, &RegEdx
);
135 if ((RegEdx
& BIT20
) != 0) {
137 // Bit 20: Execute Disable Bit available.
147 Check if Execute Disable Bit (IA32_EFER.NXE) should be enabled or not.
149 @retval TRUE IA32_EFER.NXE should be enabled.
150 @retval FALSE IA32_EFER.NXE should not be enabled.
154 IsEnableNonExecNeeded (
158 if (!IsExecuteDisableBitAvailable ()) {
163 // XD flag (BIT63) in page table entry is only valid if IA32_EFER.NXE is set.
164 // Features controlled by Following PCDs need this feature to be enabled.
166 return (PcdGetBool (PcdSetNxForStack
) ||
167 PcdGet64 (PcdDxeNxMemoryProtectionPolicy
) != 0 ||
168 PcdGet32 (PcdImageProtectionPolicy
) != 0);
172 Enable Execute Disable Bit.
176 EnableExecuteDisableBit (
182 MsrRegisters
= AsmReadMsr64 (0xC0000080);
183 if ((MsrRegisters
& BIT11
) == 0) {
184 MsrRegisters
|= BIT11
;
185 AsmWriteMsr64 (0xC0000080, MsrRegisters
);
190 The function will check if page table entry should be splitted to smaller
193 @param Address Physical memory address.
194 @param Size Size of the given physical memory.
195 @param StackBase Base address of stack.
196 @param StackSize Size of stack.
197 @param GhcbBase Base address of GHCB pages.
198 @param GhcbSize Size of GHCB area.
200 @retval TRUE Page table should be split.
201 @retval FALSE Page table should not be split.
205 IN EFI_PHYSICAL_ADDRESS Address
,
207 IN EFI_PHYSICAL_ADDRESS StackBase
,
209 IN EFI_PHYSICAL_ADDRESS GhcbBase
,
213 if (IsNullDetectionEnabled () && (Address
== 0)) {
217 if (PcdGetBool (PcdCpuStackGuard
)) {
218 if ((StackBase
>= Address
) && (StackBase
< (Address
+ Size
))) {
223 if (PcdGetBool (PcdSetNxForStack
)) {
224 if ((Address
< StackBase
+ StackSize
) && ((Address
+ Size
) > StackBase
)) {
230 if ((Address
< GhcbBase
+ GhcbSize
) && ((Address
+ Size
) > GhcbBase
)) {
239 Initialize a buffer pool for page table use only.
241 To reduce the potential split operation on page table, the pages reserved for
242 page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
243 at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
244 initialized with number of pages greater than or equal to the given PoolPages.
246 Once the pages in the pool are used up, this method should be called again to
247 reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. But usually this won't
250 @param PoolPages The least page number of the pool to be created.
252 @retval TRUE The pool is initialized successfully.
253 @retval FALSE The memory is out of resource.
256 InitializePageTablePool (
263 // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
266 PoolPages
+= 1; // Add one page for header.
267 PoolPages
= ((PoolPages
- 1) / PAGE_TABLE_POOL_UNIT_PAGES
+ 1) *
268 PAGE_TABLE_POOL_UNIT_PAGES
;
269 Buffer
= AllocateAlignedPages (PoolPages
, PAGE_TABLE_POOL_ALIGNMENT
);
270 if (Buffer
== NULL
) {
271 DEBUG ((DEBUG_ERROR
, "ERROR: Out of aligned pages\r\n"));
276 // Link all pools into a list for easier track later.
278 if (mPageTablePool
== NULL
) {
279 mPageTablePool
= Buffer
;
280 mPageTablePool
->NextPool
= mPageTablePool
;
282 ((PAGE_TABLE_POOL
*)Buffer
)->NextPool
= mPageTablePool
->NextPool
;
283 mPageTablePool
->NextPool
= Buffer
;
284 mPageTablePool
= Buffer
;
288 // Reserve one page for pool header.
290 mPageTablePool
->FreePages
= PoolPages
- 1;
291 mPageTablePool
->Offset
= EFI_PAGES_TO_SIZE (1);
297 This API provides a way to allocate memory for page table.
299 This API can be called more than once to allocate memory for page tables.
301 Allocates the number of 4KB pages and returns a pointer to the allocated
302 buffer. The buffer returned is aligned on a 4KB boundary.
304 If Pages is 0, then NULL is returned.
305 If there is not enough memory remaining to satisfy the request, then NULL is
308 @param Pages The number of 4 KB pages to allocate.
310 @return A pointer to the allocated buffer or NULL if allocation fails.
314 AllocatePageTableMemory (
325 // Renew the pool if necessary.
327 if ((mPageTablePool
== NULL
) ||
328 (Pages
> mPageTablePool
->FreePages
))
330 if (!InitializePageTablePool (Pages
)) {
335 Buffer
= (UINT8
*)mPageTablePool
+ mPageTablePool
->Offset
;
337 mPageTablePool
->Offset
+= EFI_PAGES_TO_SIZE (Pages
);
338 mPageTablePool
->FreePages
-= Pages
;
346 @param[in] PhysicalAddress Start physical address the 2M page covered.
347 @param[in, out] PageEntry2M Pointer to 2M page entry.
348 @param[in] StackBase Stack base address.
349 @param[in] StackSize Stack size.
350 @param[in] GhcbBase GHCB page area base address.
351 @param[in] GhcbSize GHCB page area size.
356 IN EFI_PHYSICAL_ADDRESS PhysicalAddress
,
357 IN OUT UINT64
*PageEntry2M
,
358 IN EFI_PHYSICAL_ADDRESS StackBase
,
360 IN EFI_PHYSICAL_ADDRESS GhcbBase
,
364 EFI_PHYSICAL_ADDRESS PhysicalAddress4K
;
365 UINTN IndexOfPageTableEntries
;
366 PAGE_TABLE_4K_ENTRY
*PageTableEntry
;
367 UINT64 AddressEncMask
;
370 // Make sure AddressEncMask is contained to smallest supported address field
372 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
374 PageTableEntry
= AllocatePageTableMemory (1);
375 ASSERT (PageTableEntry
!= NULL
);
378 // Fill in 2M page entry.
380 *PageEntry2M
= (UINT64
)(UINTN
)PageTableEntry
| AddressEncMask
| IA32_PG_P
| IA32_PG_RW
;
382 PhysicalAddress4K
= PhysicalAddress
;
383 for (IndexOfPageTableEntries
= 0; IndexOfPageTableEntries
< 512; IndexOfPageTableEntries
++, PageTableEntry
++, PhysicalAddress4K
+= SIZE_4KB
) {
385 // Fill in the Page Table entries
387 PageTableEntry
->Uint64
= (UINT64
)PhysicalAddress4K
;
390 // The GHCB range consists of two pages per CPU, the GHCB and a
391 // per-CPU variable page. The GHCB page needs to be mapped as an
392 // unencrypted page while the per-CPU variable page needs to be
393 // mapped encrypted. These pages alternate in assignment.
396 || (PhysicalAddress4K
< GhcbBase
)
397 || (PhysicalAddress4K
>= GhcbBase
+ GhcbSize
)
398 || (((PhysicalAddress4K
- GhcbBase
) & SIZE_4KB
) != 0))
400 PageTableEntry
->Uint64
|= AddressEncMask
;
403 PageTableEntry
->Bits
.ReadWrite
= 1;
405 if ((IsNullDetectionEnabled () && (PhysicalAddress4K
== 0)) ||
406 (PcdGetBool (PcdCpuStackGuard
) && (PhysicalAddress4K
== StackBase
)))
408 PageTableEntry
->Bits
.Present
= 0;
410 PageTableEntry
->Bits
.Present
= 1;
413 if ( PcdGetBool (PcdSetNxForStack
)
414 && (PhysicalAddress4K
>= StackBase
)
415 && (PhysicalAddress4K
< StackBase
+ StackSize
))
418 // Set Nx bit for stack.
420 PageTableEntry
->Bits
.Nx
= 1;
428 @param[in] PhysicalAddress Start physical address the 1G page covered.
429 @param[in, out] PageEntry1G Pointer to 1G page entry.
430 @param[in] StackBase Stack base address.
431 @param[in] StackSize Stack size.
432 @param[in] GhcbBase GHCB page area base address.
433 @param[in] GhcbSize GHCB page area size.
438 IN EFI_PHYSICAL_ADDRESS PhysicalAddress
,
439 IN OUT UINT64
*PageEntry1G
,
440 IN EFI_PHYSICAL_ADDRESS StackBase
,
442 IN EFI_PHYSICAL_ADDRESS GhcbBase
,
446 EFI_PHYSICAL_ADDRESS PhysicalAddress2M
;
447 UINTN IndexOfPageDirectoryEntries
;
448 PAGE_TABLE_ENTRY
*PageDirectoryEntry
;
449 UINT64 AddressEncMask
;
452 // Make sure AddressEncMask is contained to smallest supported address field
454 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
456 PageDirectoryEntry
= AllocatePageTableMemory (1);
457 ASSERT (PageDirectoryEntry
!= NULL
);
460 // Fill in 1G page entry.
462 *PageEntry1G
= (UINT64
)(UINTN
)PageDirectoryEntry
| AddressEncMask
| IA32_PG_P
| IA32_PG_RW
;
464 PhysicalAddress2M
= PhysicalAddress
;
465 for (IndexOfPageDirectoryEntries
= 0; IndexOfPageDirectoryEntries
< 512; IndexOfPageDirectoryEntries
++, PageDirectoryEntry
++, PhysicalAddress2M
+= SIZE_2MB
) {
466 if (ToSplitPageTable (PhysicalAddress2M
, SIZE_2MB
, StackBase
, StackSize
, GhcbBase
, GhcbSize
)) {
468 // Need to split this 2M page that covers NULL or stack range.
470 Split2MPageTo4K (PhysicalAddress2M
, (UINT64
*)PageDirectoryEntry
, StackBase
, StackSize
, GhcbBase
, GhcbSize
);
473 // Fill in the Page Directory entries
475 PageDirectoryEntry
->Uint64
= (UINT64
)PhysicalAddress2M
| AddressEncMask
;
476 PageDirectoryEntry
->Bits
.ReadWrite
= 1;
477 PageDirectoryEntry
->Bits
.Present
= 1;
478 PageDirectoryEntry
->Bits
.MustBe1
= 1;
484 Set one page of page table pool memory to be read-only.
486 @param[in] PageTableBase Base address of page table (CR3).
487 @param[in] Address Start address of a page to be set as read-only.
488 @param[in] Level4Paging Level 4 paging flag.
492 SetPageTablePoolReadOnly (
493 IN UINTN PageTableBase
,
494 IN EFI_PHYSICAL_ADDRESS Address
,
495 IN BOOLEAN Level4Paging
500 UINT64 AddressEncMask
;
501 EFI_PHYSICAL_ADDRESS PhysicalAddress
;
503 UINT64
*NewPageTable
;
511 ASSERT (PageTableBase
!= 0);
514 // Since the page table is always from page table pool, which is always
515 // located at the boundary of PcdPageTablePoolAlignment, we just need to
516 // set the whole pool unit to be read-only.
518 Address
= Address
& PAGE_TABLE_POOL_ALIGN_MASK
;
520 LevelShift
[1] = PAGING_L1_ADDRESS_SHIFT
;
521 LevelShift
[2] = PAGING_L2_ADDRESS_SHIFT
;
522 LevelShift
[3] = PAGING_L3_ADDRESS_SHIFT
;
523 LevelShift
[4] = PAGING_L4_ADDRESS_SHIFT
;
525 LevelMask
[1] = PAGING_4K_ADDRESS_MASK_64
;
526 LevelMask
[2] = PAGING_2M_ADDRESS_MASK_64
;
527 LevelMask
[3] = PAGING_1G_ADDRESS_MASK_64
;
528 LevelMask
[4] = PAGING_1G_ADDRESS_MASK_64
;
530 LevelSize
[1] = SIZE_4KB
;
531 LevelSize
[2] = SIZE_2MB
;
532 LevelSize
[3] = SIZE_1GB
;
533 LevelSize
[4] = SIZE_512GB
;
535 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) &
536 PAGING_1G_ADDRESS_MASK_64
;
537 PageTable
= (UINT64
*)(UINTN
)PageTableBase
;
538 PoolUnitSize
= PAGE_TABLE_POOL_UNIT_SIZE
;
540 for (Level
= (Level4Paging
) ? 4 : 3; Level
> 0; --Level
) {
541 Index
= ((UINTN
)RShiftU64 (Address
, LevelShift
[Level
]));
542 Index
&= PAGING_PAE_INDEX_MASK
;
544 PageAttr
= PageTable
[Index
];
545 if ((PageAttr
& IA32_PG_PS
) == 0) {
547 // Go to next level of table.
549 PageTable
= (UINT64
*)(UINTN
)(PageAttr
& ~AddressEncMask
&
550 PAGING_4K_ADDRESS_MASK_64
);
554 if (PoolUnitSize
>= LevelSize
[Level
]) {
556 // Clear R/W bit if current page granularity is not larger than pool unit
559 if ((PageAttr
& IA32_PG_RW
) != 0) {
560 while (PoolUnitSize
> 0) {
562 // PAGE_TABLE_POOL_UNIT_SIZE and PAGE_TABLE_POOL_ALIGNMENT are fit in
563 // one page (2MB). Then we don't need to update attributes for pages
564 // crossing page directory. ASSERT below is for that purpose.
566 ASSERT (Index
< EFI_PAGE_SIZE
/sizeof (UINT64
));
568 PageTable
[Index
] &= ~(UINT64
)IA32_PG_RW
;
569 PoolUnitSize
-= LevelSize
[Level
];
578 // The smaller granularity of page must be needed.
582 NewPageTable
= AllocatePageTableMemory (1);
583 ASSERT (NewPageTable
!= NULL
);
585 PhysicalAddress
= PageAttr
& LevelMask
[Level
];
587 EntryIndex
< EFI_PAGE_SIZE
/sizeof (UINT64
);
590 NewPageTable
[EntryIndex
] = PhysicalAddress
| AddressEncMask
|
591 IA32_PG_P
| IA32_PG_RW
;
593 NewPageTable
[EntryIndex
] |= IA32_PG_PS
;
596 PhysicalAddress
+= LevelSize
[Level
- 1];
599 PageTable
[Index
] = (UINT64
)(UINTN
)NewPageTable
| AddressEncMask
|
600 IA32_PG_P
| IA32_PG_RW
;
601 PageTable
= NewPageTable
;
607 Prevent the memory pages used for page table from been overwritten.
609 @param[in] PageTableBase Base address of page table (CR3).
610 @param[in] Level4Paging Level 4 paging flag.
614 EnablePageTableProtection (
615 IN UINTN PageTableBase
,
616 IN BOOLEAN Level4Paging
619 PAGE_TABLE_POOL
*HeadPool
;
620 PAGE_TABLE_POOL
*Pool
;
622 EFI_PHYSICAL_ADDRESS Address
;
624 if (mPageTablePool
== NULL
) {
629 // No need to clear CR0.WP since PageTableBase has't been written to CR3 yet.
630 // SetPageTablePoolReadOnly might update mPageTablePool. It's safer to
631 // remember original one in advance.
633 HeadPool
= mPageTablePool
;
636 Address
= (EFI_PHYSICAL_ADDRESS
)(UINTN
)Pool
;
637 PoolSize
= Pool
->Offset
+ EFI_PAGES_TO_SIZE (Pool
->FreePages
);
640 // The size of one pool must be multiple of PAGE_TABLE_POOL_UNIT_SIZE, which
641 // is one of page size of the processor (2MB by default). Let's apply the
642 // protection to them one by one.
644 while (PoolSize
> 0) {
645 SetPageTablePoolReadOnly (PageTableBase
, Address
, Level4Paging
);
646 Address
+= PAGE_TABLE_POOL_UNIT_SIZE
;
647 PoolSize
-= PAGE_TABLE_POOL_UNIT_SIZE
;
650 Pool
= Pool
->NextPool
;
651 } while (Pool
!= HeadPool
);
654 // Enable write protection, after page table attribute updated.
656 AsmWriteCr0 (AsmReadCr0 () | CR0_WP
);
660 Allocates and fills in the Page Directory and Page Table Entries to
661 establish a 1:1 Virtual to Physical mapping.
663 @param[in] StackBase Stack base address.
664 @param[in] StackSize Stack size.
665 @param[in] GhcbBase GHCB base address.
666 @param[in] GhcbSize GHCB size.
668 @return The address of 4 level page map.
672 CreateIdentityMappingPageTables (
673 IN EFI_PHYSICAL_ADDRESS StackBase
,
675 IN EFI_PHYSICAL_ADDRESS GhcbBase
,
680 CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS_ECX EcxFlags
;
682 UINT8 PhysicalAddressBits
;
683 EFI_PHYSICAL_ADDRESS PageAddress
;
684 UINTN IndexOfPml5Entries
;
685 UINTN IndexOfPml4Entries
;
686 UINTN IndexOfPdpEntries
;
687 UINTN IndexOfPageDirectoryEntries
;
688 UINT32 NumberOfPml5EntriesNeeded
;
689 UINT32 NumberOfPml4EntriesNeeded
;
690 UINT32 NumberOfPdpEntriesNeeded
;
691 PAGE_MAP_AND_DIRECTORY_POINTER
*PageMapLevel5Entry
;
692 PAGE_MAP_AND_DIRECTORY_POINTER
*PageMapLevel4Entry
;
693 PAGE_MAP_AND_DIRECTORY_POINTER
*PageMap
;
694 PAGE_MAP_AND_DIRECTORY_POINTER
*PageDirectoryPointerEntry
;
695 PAGE_TABLE_ENTRY
*PageDirectoryEntry
;
697 UINTN BigPageAddress
;
699 BOOLEAN Page5LevelSupport
;
700 BOOLEAN Page1GSupport
;
701 PAGE_TABLE_1G_ENTRY
*PageDirectory1GEntry
;
702 UINT64 AddressEncMask
;
706 // Set PageMapLevel5Entry to suppress incorrect compiler/analyzer warnings
708 PageMapLevel5Entry
= NULL
;
711 // Make sure AddressEncMask is contained to smallest supported address field
713 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
715 Page1GSupport
= FALSE
;
716 if (PcdGetBool (PcdUse1GPageTable
)) {
717 AsmCpuid (0x80000000, &RegEax
, NULL
, NULL
, NULL
);
718 if (RegEax
>= 0x80000001) {
719 AsmCpuid (0x80000001, NULL
, NULL
, NULL
, &RegEdx
);
720 if ((RegEdx
& BIT26
) != 0) {
721 Page1GSupport
= TRUE
;
727 // Get physical address bits supported.
729 Hob
= GetFirstHob (EFI_HOB_TYPE_CPU
);
731 PhysicalAddressBits
= ((EFI_HOB_CPU
*)Hob
)->SizeOfMemorySpace
;
733 AsmCpuid (0x80000000, &RegEax
, NULL
, NULL
, NULL
);
734 if (RegEax
>= 0x80000008) {
735 AsmCpuid (0x80000008, &RegEax
, NULL
, NULL
, NULL
);
736 PhysicalAddressBits
= (UINT8
)RegEax
;
738 PhysicalAddressBits
= 36;
742 Page5LevelSupport
= FALSE
;
743 if (PcdGetBool (PcdUse5LevelPageTable
)) {
745 CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS
,
746 CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS_SUB_LEAF_INFO
,
752 if (EcxFlags
.Bits
.FiveLevelPage
!= 0) {
753 Page5LevelSupport
= TRUE
;
757 DEBUG ((DEBUG_INFO
, "AddressBits=%u 5LevelPaging=%u 1GPage=%u\n", PhysicalAddressBits
, Page5LevelSupport
, Page1GSupport
));
760 // IA-32e paging translates 48-bit linear addresses to 52-bit physical addresses
761 // when 5-Level Paging is disabled,
762 // due to either unsupported by HW, or disabled by PCD.
764 ASSERT (PhysicalAddressBits
<= 52);
765 if (!Page5LevelSupport
&& (PhysicalAddressBits
> 48)) {
766 PhysicalAddressBits
= 48;
770 // Calculate the table entries needed.
772 NumberOfPml5EntriesNeeded
= 1;
773 if (PhysicalAddressBits
> 48) {
774 NumberOfPml5EntriesNeeded
= (UINT32
)LShiftU64 (1, PhysicalAddressBits
- 48);
775 PhysicalAddressBits
= 48;
778 NumberOfPml4EntriesNeeded
= 1;
779 if (PhysicalAddressBits
> 39) {
780 NumberOfPml4EntriesNeeded
= (UINT32
)LShiftU64 (1, PhysicalAddressBits
- 39);
781 PhysicalAddressBits
= 39;
784 NumberOfPdpEntriesNeeded
= 1;
785 ASSERT (PhysicalAddressBits
> 30);
786 NumberOfPdpEntriesNeeded
= (UINT32
)LShiftU64 (1, PhysicalAddressBits
- 30);
789 // Pre-allocate big pages to avoid later allocations.
791 if (!Page1GSupport
) {
792 TotalPagesNum
= ((NumberOfPdpEntriesNeeded
+ 1) * NumberOfPml4EntriesNeeded
+ 1) * NumberOfPml5EntriesNeeded
+ 1;
794 TotalPagesNum
= (NumberOfPml4EntriesNeeded
+ 1) * NumberOfPml5EntriesNeeded
+ 1;
798 // Substract the one page occupied by PML5 entries if 5-Level Paging is disabled.
800 if (!Page5LevelSupport
) {
806 "Pml5=%u Pml4=%u Pdp=%u TotalPage=%Lu\n",
807 NumberOfPml5EntriesNeeded
,
808 NumberOfPml4EntriesNeeded
,
809 NumberOfPdpEntriesNeeded
,
810 (UINT64
)TotalPagesNum
813 BigPageAddress
= (UINTN
)AllocatePageTableMemory (TotalPagesNum
);
814 ASSERT (BigPageAddress
!= 0);
817 // By architecture only one PageMapLevel4 exists - so lets allocate storage for it.
819 PageMap
= (VOID
*)BigPageAddress
;
820 if (Page5LevelSupport
) {
822 // By architecture only one PageMapLevel5 exists - so lets allocate storage for it.
824 PageMapLevel5Entry
= PageMap
;
825 BigPageAddress
+= SIZE_4KB
;
830 for ( IndexOfPml5Entries
= 0
831 ; IndexOfPml5Entries
< NumberOfPml5EntriesNeeded
832 ; IndexOfPml5Entries
++)
835 // Each PML5 entry points to a page of PML4 entires.
836 // So lets allocate space for them and fill them in in the IndexOfPml4Entries loop.
837 // When 5-Level Paging is disabled, below allocation happens only once.
839 PageMapLevel4Entry
= (VOID
*)BigPageAddress
;
840 BigPageAddress
+= SIZE_4KB
;
842 if (Page5LevelSupport
) {
846 PageMapLevel5Entry
->Uint64
= (UINT64
)(UINTN
)PageMapLevel4Entry
| AddressEncMask
;
847 PageMapLevel5Entry
->Bits
.ReadWrite
= 1;
848 PageMapLevel5Entry
->Bits
.Present
= 1;
849 PageMapLevel5Entry
++;
852 for ( IndexOfPml4Entries
= 0
853 ; IndexOfPml4Entries
< (NumberOfPml5EntriesNeeded
== 1 ? NumberOfPml4EntriesNeeded
: 512)
854 ; IndexOfPml4Entries
++, PageMapLevel4Entry
++)
857 // Each PML4 entry points to a page of Page Directory Pointer entires.
858 // So lets allocate space for them and fill them in in the IndexOfPdpEntries loop.
860 PageDirectoryPointerEntry
= (VOID
*)BigPageAddress
;
861 BigPageAddress
+= SIZE_4KB
;
866 PageMapLevel4Entry
->Uint64
= (UINT64
)(UINTN
)PageDirectoryPointerEntry
| AddressEncMask
;
867 PageMapLevel4Entry
->Bits
.ReadWrite
= 1;
868 PageMapLevel4Entry
->Bits
.Present
= 1;
871 PageDirectory1GEntry
= (VOID
*)PageDirectoryPointerEntry
;
873 for (IndexOfPageDirectoryEntries
= 0; IndexOfPageDirectoryEntries
< 512; IndexOfPageDirectoryEntries
++, PageDirectory1GEntry
++, PageAddress
+= SIZE_1GB
) {
874 if (ToSplitPageTable (PageAddress
, SIZE_1GB
, StackBase
, StackSize
, GhcbBase
, GhcbSize
)) {
875 Split1GPageTo2M (PageAddress
, (UINT64
*)PageDirectory1GEntry
, StackBase
, StackSize
, GhcbBase
, GhcbSize
);
878 // Fill in the Page Directory entries
880 PageDirectory1GEntry
->Uint64
= (UINT64
)PageAddress
| AddressEncMask
;
881 PageDirectory1GEntry
->Bits
.ReadWrite
= 1;
882 PageDirectory1GEntry
->Bits
.Present
= 1;
883 PageDirectory1GEntry
->Bits
.MustBe1
= 1;
887 for ( IndexOfPdpEntries
= 0
888 ; IndexOfPdpEntries
< (NumberOfPml4EntriesNeeded
== 1 ? NumberOfPdpEntriesNeeded
: 512)
889 ; IndexOfPdpEntries
++, PageDirectoryPointerEntry
++)
892 // Each Directory Pointer entries points to a page of Page Directory entires.
893 // So allocate space for them and fill them in in the IndexOfPageDirectoryEntries loop.
895 PageDirectoryEntry
= (VOID
*)BigPageAddress
;
896 BigPageAddress
+= SIZE_4KB
;
899 // Fill in a Page Directory Pointer Entries
901 PageDirectoryPointerEntry
->Uint64
= (UINT64
)(UINTN
)PageDirectoryEntry
| AddressEncMask
;
902 PageDirectoryPointerEntry
->Bits
.ReadWrite
= 1;
903 PageDirectoryPointerEntry
->Bits
.Present
= 1;
905 for (IndexOfPageDirectoryEntries
= 0; IndexOfPageDirectoryEntries
< 512; IndexOfPageDirectoryEntries
++, PageDirectoryEntry
++, PageAddress
+= SIZE_2MB
) {
906 if (ToSplitPageTable (PageAddress
, SIZE_2MB
, StackBase
, StackSize
, GhcbBase
, GhcbSize
)) {
908 // Need to split this 2M page that covers NULL or stack range.
910 Split2MPageTo4K (PageAddress
, (UINT64
*)PageDirectoryEntry
, StackBase
, StackSize
, GhcbBase
, GhcbSize
);
913 // Fill in the Page Directory entries
915 PageDirectoryEntry
->Uint64
= (UINT64
)PageAddress
| AddressEncMask
;
916 PageDirectoryEntry
->Bits
.ReadWrite
= 1;
917 PageDirectoryEntry
->Bits
.Present
= 1;
918 PageDirectoryEntry
->Bits
.MustBe1
= 1;
924 // Fill with null entry for unused PDPTE
926 ZeroMem (PageDirectoryPointerEntry
, (512 - IndexOfPdpEntries
) * sizeof (PAGE_MAP_AND_DIRECTORY_POINTER
));
931 // For the PML4 entries we are not using fill in a null entry.
933 ZeroMem (PageMapLevel4Entry
, (512 - IndexOfPml4Entries
) * sizeof (PAGE_MAP_AND_DIRECTORY_POINTER
));
936 if (Page5LevelSupport
) {
937 Cr4
.UintN
= AsmReadCr4 ();
939 AsmWriteCr4 (Cr4
.UintN
);
941 // For the PML5 entries we are not using fill in a null entry.
943 ZeroMem (PageMapLevel5Entry
, (512 - IndexOfPml5Entries
) * sizeof (PAGE_MAP_AND_DIRECTORY_POINTER
));
947 // Protect the page table by marking the memory used for page table to be
950 EnablePageTableProtection ((UINTN
)PageMap
, TRUE
);
953 // Set IA32_EFER.NXE if necessary.
955 if (IsEnableNonExecNeeded ()) {
956 EnableExecuteDisableBit ();
959 return (UINTN
)PageMap
;