2 x64 Virtual Memory Management Services in the form of an IA-32 driver.
3 Used to establish a 1:1 Virtual to Physical Mapping that is required to
4 enter Long Mode (x64 64-bit mode).
6 While we make a 1:1 mapping (identity mapping) for all physical pages
7 we still need to use the MTRR's to ensure that the cachability attributes
8 for all memory regions is correct.
10 The basic idea is to use 2MB page table entries where ever possible. If
11 more granularity of cachability is required then 4K page tables are used.
14 1) IA-32 Intel(R) Architecture Software Developer's Manual Volume 1:Basic Architecture, Intel
15 2) IA-32 Intel(R) Architecture Software Developer's Manual Volume 2:Instruction Set Reference, Intel
16 3) IA-32 Intel(R) Architecture Software Developer's Manual Volume 3:System Programmer's Guide, Intel
18 Copyright (c) 2006 - 2019, Intel Corporation. All rights reserved.<BR>
19 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
21 SPDX-License-Identifier: BSD-2-Clause-Patent
25 #include <Register/Intel/Cpuid.h>
27 #include "VirtualMemory.h"
30 // Global variable to keep track current available memory used as page table.
32 PAGE_TABLE_POOL
*mPageTablePool
= NULL
;
35 Clear legacy memory located at the first 4K-page, if available.
37 This function traverses the whole HOB list to check if memory from 0 to 4095
38 exists and has not been allocated, and then clear it if so.
40 @param HobStart The start of HobList passed to DxeCore.
48 EFI_PEI_HOB_POINTERS RscHob
;
49 EFI_PEI_HOB_POINTERS MemHob
;
52 RscHob
.Raw
= HobStart
;
53 MemHob
.Raw
= HobStart
;
57 // Check if page 0 exists and free
59 while ((RscHob
.Raw
= GetNextHob (
60 EFI_HOB_TYPE_RESOURCE_DESCRIPTOR
,
64 if ((RscHob
.ResourceDescriptor
->ResourceType
== EFI_RESOURCE_SYSTEM_MEMORY
) &&
65 (RscHob
.ResourceDescriptor
->PhysicalStart
== 0))
69 // Make sure memory at 0-4095 has not been allocated.
71 while ((MemHob
.Raw
= GetNextHob (
72 EFI_HOB_TYPE_MEMORY_ALLOCATION
,
76 if (MemHob
.MemoryAllocation
->AllocDescriptor
.MemoryBaseAddress
83 MemHob
.Raw
= GET_NEXT_HOB (MemHob
);
89 RscHob
.Raw
= GET_NEXT_HOB (RscHob
);
93 DEBUG ((DEBUG_INFO
, "Clearing first 4K-page!\r\n"));
94 SetMem (NULL
, EFI_PAGE_SIZE
, 0);
101 Return configure status of NULL pointer detection feature.
103 @return TRUE NULL pointer detection feature is enabled
104 @return FALSE NULL pointer detection feature is disabled
108 IsNullDetectionEnabled (
112 return ((PcdGet8 (PcdNullPointerDetectionPropertyMask
) & BIT0
) != 0);
116 The function will check if Execute Disable Bit is available.
118 @retval TRUE Execute Disable Bit is available.
119 @retval FALSE Execute Disable Bit is not available.
123 IsExecuteDisableBitAvailable (
132 AsmCpuid (0x80000000, &RegEax
, NULL
, NULL
, NULL
);
133 if (RegEax
>= 0x80000001) {
134 AsmCpuid (0x80000001, NULL
, NULL
, NULL
, &RegEdx
);
135 if ((RegEdx
& BIT20
) != 0) {
137 // Bit 20: Execute Disable Bit available.
147 Check if Execute Disable Bit (IA32_EFER.NXE) should be enabled or not.
149 @retval TRUE IA32_EFER.NXE should be enabled.
150 @retval FALSE IA32_EFER.NXE should not be enabled.
154 IsEnableNonExecNeeded (
158 if (!IsExecuteDisableBitAvailable ()) {
163 // XD flag (BIT63) in page table entry is only valid if IA32_EFER.NXE is set.
164 // Features controlled by Following PCDs need this feature to be enabled.
166 return (PcdGetBool (PcdSetNxForStack
) ||
167 PcdGet64 (PcdDxeNxMemoryProtectionPolicy
) != 0 ||
168 PcdGet32 (PcdImageProtectionPolicy
) != 0);
172 Enable Execute Disable Bit.
176 EnableExecuteDisableBit (
182 MsrRegisters
= AsmReadMsr64 (0xC0000080);
183 MsrRegisters
|= BIT11
;
184 AsmWriteMsr64 (0xC0000080, MsrRegisters
);
188 The function will check if page table entry should be splitted to smaller
191 @param Address Physical memory address.
192 @param Size Size of the given physical memory.
193 @param StackBase Base address of stack.
194 @param StackSize Size of stack.
195 @param GhcbBase Base address of GHCB pages.
196 @param GhcbSize Size of GHCB area.
198 @retval TRUE Page table should be split.
199 @retval FALSE Page table should not be split.
203 IN EFI_PHYSICAL_ADDRESS Address
,
205 IN EFI_PHYSICAL_ADDRESS StackBase
,
207 IN EFI_PHYSICAL_ADDRESS GhcbBase
,
211 if (IsNullDetectionEnabled () && (Address
== 0)) {
215 if (PcdGetBool (PcdCpuStackGuard
)) {
216 if ((StackBase
>= Address
) && (StackBase
< (Address
+ Size
))) {
221 if (PcdGetBool (PcdSetNxForStack
)) {
222 if ((Address
< StackBase
+ StackSize
) && ((Address
+ Size
) > StackBase
)) {
228 if ((Address
< GhcbBase
+ GhcbSize
) && ((Address
+ Size
) > GhcbBase
)) {
237 Initialize a buffer pool for page table use only.
239 To reduce the potential split operation on page table, the pages reserved for
240 page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
241 at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
242 initialized with number of pages greater than or equal to the given PoolPages.
244 Once the pages in the pool are used up, this method should be called again to
245 reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. But usually this won't
248 @param PoolPages The least page number of the pool to be created.
250 @retval TRUE The pool is initialized successfully.
251 @retval FALSE The memory is out of resource.
254 InitializePageTablePool (
261 // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
264 PoolPages
+= 1; // Add one page for header.
265 PoolPages
= ((PoolPages
- 1) / PAGE_TABLE_POOL_UNIT_PAGES
+ 1) *
266 PAGE_TABLE_POOL_UNIT_PAGES
;
267 Buffer
= AllocateAlignedPages (PoolPages
, PAGE_TABLE_POOL_ALIGNMENT
);
268 if (Buffer
== NULL
) {
269 DEBUG ((DEBUG_ERROR
, "ERROR: Out of aligned pages\r\n"));
274 // Link all pools into a list for easier track later.
276 if (mPageTablePool
== NULL
) {
277 mPageTablePool
= Buffer
;
278 mPageTablePool
->NextPool
= mPageTablePool
;
280 ((PAGE_TABLE_POOL
*)Buffer
)->NextPool
= mPageTablePool
->NextPool
;
281 mPageTablePool
->NextPool
= Buffer
;
282 mPageTablePool
= Buffer
;
286 // Reserve one page for pool header.
288 mPageTablePool
->FreePages
= PoolPages
- 1;
289 mPageTablePool
->Offset
= EFI_PAGES_TO_SIZE (1);
295 This API provides a way to allocate memory for page table.
297 This API can be called more than once to allocate memory for page tables.
299 Allocates the number of 4KB pages and returns a pointer to the allocated
300 buffer. The buffer returned is aligned on a 4KB boundary.
302 If Pages is 0, then NULL is returned.
303 If there is not enough memory remaining to satisfy the request, then NULL is
306 @param Pages The number of 4 KB pages to allocate.
308 @return A pointer to the allocated buffer or NULL if allocation fails.
312 AllocatePageTableMemory (
323 // Renew the pool if necessary.
325 if ((mPageTablePool
== NULL
) ||
326 (Pages
> mPageTablePool
->FreePages
))
328 if (!InitializePageTablePool (Pages
)) {
333 Buffer
= (UINT8
*)mPageTablePool
+ mPageTablePool
->Offset
;
335 mPageTablePool
->Offset
+= EFI_PAGES_TO_SIZE (Pages
);
336 mPageTablePool
->FreePages
-= Pages
;
344 @param[in] PhysicalAddress Start physical address the 2M page covered.
345 @param[in, out] PageEntry2M Pointer to 2M page entry.
346 @param[in] StackBase Stack base address.
347 @param[in] StackSize Stack size.
348 @param[in] GhcbBase GHCB page area base address.
349 @param[in] GhcbSize GHCB page area size.
354 IN EFI_PHYSICAL_ADDRESS PhysicalAddress
,
355 IN OUT UINT64
*PageEntry2M
,
356 IN EFI_PHYSICAL_ADDRESS StackBase
,
358 IN EFI_PHYSICAL_ADDRESS GhcbBase
,
362 EFI_PHYSICAL_ADDRESS PhysicalAddress4K
;
363 UINTN IndexOfPageTableEntries
;
364 PAGE_TABLE_4K_ENTRY
*PageTableEntry
;
365 UINT64 AddressEncMask
;
368 // Make sure AddressEncMask is contained to smallest supported address field
370 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
372 PageTableEntry
= AllocatePageTableMemory (1);
373 ASSERT (PageTableEntry
!= NULL
);
376 // Fill in 2M page entry.
378 *PageEntry2M
= (UINT64
)(UINTN
)PageTableEntry
| AddressEncMask
| IA32_PG_P
| IA32_PG_RW
;
380 PhysicalAddress4K
= PhysicalAddress
;
381 for (IndexOfPageTableEntries
= 0; IndexOfPageTableEntries
< 512; IndexOfPageTableEntries
++, PageTableEntry
++, PhysicalAddress4K
+= SIZE_4KB
) {
383 // Fill in the Page Table entries
385 PageTableEntry
->Uint64
= (UINT64
)PhysicalAddress4K
;
388 // The GHCB range consists of two pages per CPU, the GHCB and a
389 // per-CPU variable page. The GHCB page needs to be mapped as an
390 // unencrypted page while the per-CPU variable page needs to be
391 // mapped encrypted. These pages alternate in assignment.
394 || (PhysicalAddress4K
< GhcbBase
)
395 || (PhysicalAddress4K
>= GhcbBase
+ GhcbSize
)
396 || (((PhysicalAddress4K
- GhcbBase
) & SIZE_4KB
) != 0))
398 PageTableEntry
->Uint64
|= AddressEncMask
;
401 PageTableEntry
->Bits
.ReadWrite
= 1;
403 if ((IsNullDetectionEnabled () && (PhysicalAddress4K
== 0)) ||
404 (PcdGetBool (PcdCpuStackGuard
) && (PhysicalAddress4K
== StackBase
)))
406 PageTableEntry
->Bits
.Present
= 0;
408 PageTableEntry
->Bits
.Present
= 1;
411 if ( PcdGetBool (PcdSetNxForStack
)
412 && (PhysicalAddress4K
>= StackBase
)
413 && (PhysicalAddress4K
< StackBase
+ StackSize
))
416 // Set Nx bit for stack.
418 PageTableEntry
->Bits
.Nx
= 1;
426 @param[in] PhysicalAddress Start physical address the 1G page covered.
427 @param[in, out] PageEntry1G Pointer to 1G page entry.
428 @param[in] StackBase Stack base address.
429 @param[in] StackSize Stack size.
430 @param[in] GhcbBase GHCB page area base address.
431 @param[in] GhcbSize GHCB page area size.
436 IN EFI_PHYSICAL_ADDRESS PhysicalAddress
,
437 IN OUT UINT64
*PageEntry1G
,
438 IN EFI_PHYSICAL_ADDRESS StackBase
,
440 IN EFI_PHYSICAL_ADDRESS GhcbBase
,
444 EFI_PHYSICAL_ADDRESS PhysicalAddress2M
;
445 UINTN IndexOfPageDirectoryEntries
;
446 PAGE_TABLE_ENTRY
*PageDirectoryEntry
;
447 UINT64 AddressEncMask
;
450 // Make sure AddressEncMask is contained to smallest supported address field
452 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
454 PageDirectoryEntry
= AllocatePageTableMemory (1);
455 ASSERT (PageDirectoryEntry
!= NULL
);
458 // Fill in 1G page entry.
460 *PageEntry1G
= (UINT64
)(UINTN
)PageDirectoryEntry
| AddressEncMask
| IA32_PG_P
| IA32_PG_RW
;
462 PhysicalAddress2M
= PhysicalAddress
;
463 for (IndexOfPageDirectoryEntries
= 0; IndexOfPageDirectoryEntries
< 512; IndexOfPageDirectoryEntries
++, PageDirectoryEntry
++, PhysicalAddress2M
+= SIZE_2MB
) {
464 if (ToSplitPageTable (PhysicalAddress2M
, SIZE_2MB
, StackBase
, StackSize
, GhcbBase
, GhcbSize
)) {
466 // Need to split this 2M page that covers NULL or stack range.
468 Split2MPageTo4K (PhysicalAddress2M
, (UINT64
*)PageDirectoryEntry
, StackBase
, StackSize
, GhcbBase
, GhcbSize
);
471 // Fill in the Page Directory entries
473 PageDirectoryEntry
->Uint64
= (UINT64
)PhysicalAddress2M
| AddressEncMask
;
474 PageDirectoryEntry
->Bits
.ReadWrite
= 1;
475 PageDirectoryEntry
->Bits
.Present
= 1;
476 PageDirectoryEntry
->Bits
.MustBe1
= 1;
482 Set one page of page table pool memory to be read-only.
484 @param[in] PageTableBase Base address of page table (CR3).
485 @param[in] Address Start address of a page to be set as read-only.
486 @param[in] Level4Paging Level 4 paging flag.
490 SetPageTablePoolReadOnly (
491 IN UINTN PageTableBase
,
492 IN EFI_PHYSICAL_ADDRESS Address
,
493 IN BOOLEAN Level4Paging
498 UINT64 AddressEncMask
;
499 EFI_PHYSICAL_ADDRESS PhysicalAddress
;
501 UINT64
*NewPageTable
;
509 ASSERT (PageTableBase
!= 0);
512 // Since the page table is always from page table pool, which is always
513 // located at the boundary of PcdPageTablePoolAlignment, we just need to
514 // set the whole pool unit to be read-only.
516 Address
= Address
& PAGE_TABLE_POOL_ALIGN_MASK
;
518 LevelShift
[1] = PAGING_L1_ADDRESS_SHIFT
;
519 LevelShift
[2] = PAGING_L2_ADDRESS_SHIFT
;
520 LevelShift
[3] = PAGING_L3_ADDRESS_SHIFT
;
521 LevelShift
[4] = PAGING_L4_ADDRESS_SHIFT
;
523 LevelMask
[1] = PAGING_4K_ADDRESS_MASK_64
;
524 LevelMask
[2] = PAGING_2M_ADDRESS_MASK_64
;
525 LevelMask
[3] = PAGING_1G_ADDRESS_MASK_64
;
526 LevelMask
[4] = PAGING_1G_ADDRESS_MASK_64
;
528 LevelSize
[1] = SIZE_4KB
;
529 LevelSize
[2] = SIZE_2MB
;
530 LevelSize
[3] = SIZE_1GB
;
531 LevelSize
[4] = SIZE_512GB
;
533 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) &
534 PAGING_1G_ADDRESS_MASK_64
;
535 PageTable
= (UINT64
*)(UINTN
)PageTableBase
;
536 PoolUnitSize
= PAGE_TABLE_POOL_UNIT_SIZE
;
538 for (Level
= (Level4Paging
) ? 4 : 3; Level
> 0; --Level
) {
539 Index
= ((UINTN
)RShiftU64 (Address
, LevelShift
[Level
]));
540 Index
&= PAGING_PAE_INDEX_MASK
;
542 PageAttr
= PageTable
[Index
];
543 if ((PageAttr
& IA32_PG_PS
) == 0) {
545 // Go to next level of table.
547 PageTable
= (UINT64
*)(UINTN
)(PageAttr
& ~AddressEncMask
&
548 PAGING_4K_ADDRESS_MASK_64
);
552 if (PoolUnitSize
>= LevelSize
[Level
]) {
554 // Clear R/W bit if current page granularity is not larger than pool unit
557 if ((PageAttr
& IA32_PG_RW
) != 0) {
558 while (PoolUnitSize
> 0) {
560 // PAGE_TABLE_POOL_UNIT_SIZE and PAGE_TABLE_POOL_ALIGNMENT are fit in
561 // one page (2MB). Then we don't need to update attributes for pages
562 // crossing page directory. ASSERT below is for that purpose.
564 ASSERT (Index
< EFI_PAGE_SIZE
/sizeof (UINT64
));
566 PageTable
[Index
] &= ~(UINT64
)IA32_PG_RW
;
567 PoolUnitSize
-= LevelSize
[Level
];
576 // The smaller granularity of page must be needed.
580 NewPageTable
= AllocatePageTableMemory (1);
581 ASSERT (NewPageTable
!= NULL
);
583 PhysicalAddress
= PageAttr
& LevelMask
[Level
];
585 EntryIndex
< EFI_PAGE_SIZE
/sizeof (UINT64
);
588 NewPageTable
[EntryIndex
] = PhysicalAddress
| AddressEncMask
|
589 IA32_PG_P
| IA32_PG_RW
;
591 NewPageTable
[EntryIndex
] |= IA32_PG_PS
;
594 PhysicalAddress
+= LevelSize
[Level
- 1];
597 PageTable
[Index
] = (UINT64
)(UINTN
)NewPageTable
| AddressEncMask
|
598 IA32_PG_P
| IA32_PG_RW
;
599 PageTable
= NewPageTable
;
605 Prevent the memory pages used for page table from been overwritten.
607 @param[in] PageTableBase Base address of page table (CR3).
608 @param[in] Level4Paging Level 4 paging flag.
612 EnablePageTableProtection (
613 IN UINTN PageTableBase
,
614 IN BOOLEAN Level4Paging
617 PAGE_TABLE_POOL
*HeadPool
;
618 PAGE_TABLE_POOL
*Pool
;
620 EFI_PHYSICAL_ADDRESS Address
;
622 if (mPageTablePool
== NULL
) {
627 // Disable write protection, because we need to mark page table to be write
630 AsmWriteCr0 (AsmReadCr0 () & ~CR0_WP
);
633 // SetPageTablePoolReadOnly might update mPageTablePool. It's safer to
634 // remember original one in advance.
636 HeadPool
= mPageTablePool
;
639 Address
= (EFI_PHYSICAL_ADDRESS
)(UINTN
)Pool
;
640 PoolSize
= Pool
->Offset
+ EFI_PAGES_TO_SIZE (Pool
->FreePages
);
643 // The size of one pool must be multiple of PAGE_TABLE_POOL_UNIT_SIZE, which
644 // is one of page size of the processor (2MB by default). Let's apply the
645 // protection to them one by one.
647 while (PoolSize
> 0) {
648 SetPageTablePoolReadOnly (PageTableBase
, Address
, Level4Paging
);
649 Address
+= PAGE_TABLE_POOL_UNIT_SIZE
;
650 PoolSize
-= PAGE_TABLE_POOL_UNIT_SIZE
;
653 Pool
= Pool
->NextPool
;
654 } while (Pool
!= HeadPool
);
657 // Enable write protection, after page table attribute updated.
659 AsmWriteCr0 (AsmReadCr0 () | CR0_WP
);
663 Allocates and fills in the Page Directory and Page Table Entries to
664 establish a 1:1 Virtual to Physical mapping.
666 @param[in] StackBase Stack base address.
667 @param[in] StackSize Stack size.
668 @param[in] GhcbBase GHCB base address.
669 @param[in] GhcbSize GHCB size.
671 @return The address of 4 level page map.
675 CreateIdentityMappingPageTables (
676 IN EFI_PHYSICAL_ADDRESS StackBase
,
678 IN EFI_PHYSICAL_ADDRESS GhcbBase
,
683 CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS_ECX EcxFlags
;
685 UINT8 PhysicalAddressBits
;
686 EFI_PHYSICAL_ADDRESS PageAddress
;
687 UINTN IndexOfPml5Entries
;
688 UINTN IndexOfPml4Entries
;
689 UINTN IndexOfPdpEntries
;
690 UINTN IndexOfPageDirectoryEntries
;
691 UINT32 NumberOfPml5EntriesNeeded
;
692 UINT32 NumberOfPml4EntriesNeeded
;
693 UINT32 NumberOfPdpEntriesNeeded
;
694 PAGE_MAP_AND_DIRECTORY_POINTER
*PageMapLevel5Entry
;
695 PAGE_MAP_AND_DIRECTORY_POINTER
*PageMapLevel4Entry
;
696 PAGE_MAP_AND_DIRECTORY_POINTER
*PageMap
;
697 PAGE_MAP_AND_DIRECTORY_POINTER
*PageDirectoryPointerEntry
;
698 PAGE_TABLE_ENTRY
*PageDirectoryEntry
;
700 UINTN BigPageAddress
;
702 BOOLEAN Page5LevelSupport
;
703 BOOLEAN Page1GSupport
;
704 PAGE_TABLE_1G_ENTRY
*PageDirectory1GEntry
;
705 UINT64 AddressEncMask
;
709 // Set PageMapLevel5Entry to suppress incorrect compiler/analyzer warnings
711 PageMapLevel5Entry
= NULL
;
714 // Make sure AddressEncMask is contained to smallest supported address field
716 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
718 Page1GSupport
= FALSE
;
719 if (PcdGetBool (PcdUse1GPageTable
)) {
720 AsmCpuid (0x80000000, &RegEax
, NULL
, NULL
, NULL
);
721 if (RegEax
>= 0x80000001) {
722 AsmCpuid (0x80000001, NULL
, NULL
, NULL
, &RegEdx
);
723 if ((RegEdx
& BIT26
) != 0) {
724 Page1GSupport
= TRUE
;
730 // Get physical address bits supported.
732 Hob
= GetFirstHob (EFI_HOB_TYPE_CPU
);
734 PhysicalAddressBits
= ((EFI_HOB_CPU
*)Hob
)->SizeOfMemorySpace
;
736 AsmCpuid (0x80000000, &RegEax
, NULL
, NULL
, NULL
);
737 if (RegEax
>= 0x80000008) {
738 AsmCpuid (0x80000008, &RegEax
, NULL
, NULL
, NULL
);
739 PhysicalAddressBits
= (UINT8
)RegEax
;
741 PhysicalAddressBits
= 36;
745 Page5LevelSupport
= FALSE
;
746 if (PcdGetBool (PcdUse5LevelPageTable
)) {
748 CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS
,
749 CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS_SUB_LEAF_INFO
,
755 if (EcxFlags
.Bits
.FiveLevelPage
!= 0) {
756 Page5LevelSupport
= TRUE
;
760 DEBUG ((DEBUG_INFO
, "AddressBits=%u 5LevelPaging=%u 1GPage=%u\n", PhysicalAddressBits
, Page5LevelSupport
, Page1GSupport
));
763 // IA-32e paging translates 48-bit linear addresses to 52-bit physical addresses
764 // when 5-Level Paging is disabled,
765 // due to either unsupported by HW, or disabled by PCD.
767 ASSERT (PhysicalAddressBits
<= 52);
768 if (!Page5LevelSupport
&& (PhysicalAddressBits
> 48)) {
769 PhysicalAddressBits
= 48;
773 // Calculate the table entries needed.
775 NumberOfPml5EntriesNeeded
= 1;
776 if (PhysicalAddressBits
> 48) {
777 NumberOfPml5EntriesNeeded
= (UINT32
)LShiftU64 (1, PhysicalAddressBits
- 48);
778 PhysicalAddressBits
= 48;
781 NumberOfPml4EntriesNeeded
= 1;
782 if (PhysicalAddressBits
> 39) {
783 NumberOfPml4EntriesNeeded
= (UINT32
)LShiftU64 (1, PhysicalAddressBits
- 39);
784 PhysicalAddressBits
= 39;
787 NumberOfPdpEntriesNeeded
= 1;
788 ASSERT (PhysicalAddressBits
> 30);
789 NumberOfPdpEntriesNeeded
= (UINT32
)LShiftU64 (1, PhysicalAddressBits
- 30);
792 // Pre-allocate big pages to avoid later allocations.
794 if (!Page1GSupport
) {
795 TotalPagesNum
= ((NumberOfPdpEntriesNeeded
+ 1) * NumberOfPml4EntriesNeeded
+ 1) * NumberOfPml5EntriesNeeded
+ 1;
797 TotalPagesNum
= (NumberOfPml4EntriesNeeded
+ 1) * NumberOfPml5EntriesNeeded
+ 1;
801 // Substract the one page occupied by PML5 entries if 5-Level Paging is disabled.
803 if (!Page5LevelSupport
) {
809 "Pml5=%u Pml4=%u Pdp=%u TotalPage=%Lu\n",
810 NumberOfPml5EntriesNeeded
,
811 NumberOfPml4EntriesNeeded
,
812 NumberOfPdpEntriesNeeded
,
813 (UINT64
)TotalPagesNum
816 BigPageAddress
= (UINTN
)AllocatePageTableMemory (TotalPagesNum
);
817 ASSERT (BigPageAddress
!= 0);
820 // By architecture only one PageMapLevel4 exists - so lets allocate storage for it.
822 PageMap
= (VOID
*)BigPageAddress
;
823 if (Page5LevelSupport
) {
825 // By architecture only one PageMapLevel5 exists - so lets allocate storage for it.
827 PageMapLevel5Entry
= PageMap
;
828 BigPageAddress
+= SIZE_4KB
;
833 for ( IndexOfPml5Entries
= 0
834 ; IndexOfPml5Entries
< NumberOfPml5EntriesNeeded
835 ; IndexOfPml5Entries
++)
838 // Each PML5 entry points to a page of PML4 entires.
839 // So lets allocate space for them and fill them in in the IndexOfPml4Entries loop.
840 // When 5-Level Paging is disabled, below allocation happens only once.
842 PageMapLevel4Entry
= (VOID
*)BigPageAddress
;
843 BigPageAddress
+= SIZE_4KB
;
845 if (Page5LevelSupport
) {
849 PageMapLevel5Entry
->Uint64
= (UINT64
)(UINTN
)PageMapLevel4Entry
| AddressEncMask
;
850 PageMapLevel5Entry
->Bits
.ReadWrite
= 1;
851 PageMapLevel5Entry
->Bits
.Present
= 1;
852 PageMapLevel5Entry
++;
855 for ( IndexOfPml4Entries
= 0
856 ; IndexOfPml4Entries
< (NumberOfPml5EntriesNeeded
== 1 ? NumberOfPml4EntriesNeeded
: 512)
857 ; IndexOfPml4Entries
++, PageMapLevel4Entry
++)
860 // Each PML4 entry points to a page of Page Directory Pointer entires.
861 // So lets allocate space for them and fill them in in the IndexOfPdpEntries loop.
863 PageDirectoryPointerEntry
= (VOID
*)BigPageAddress
;
864 BigPageAddress
+= SIZE_4KB
;
869 PageMapLevel4Entry
->Uint64
= (UINT64
)(UINTN
)PageDirectoryPointerEntry
| AddressEncMask
;
870 PageMapLevel4Entry
->Bits
.ReadWrite
= 1;
871 PageMapLevel4Entry
->Bits
.Present
= 1;
874 PageDirectory1GEntry
= (VOID
*)PageDirectoryPointerEntry
;
876 for (IndexOfPageDirectoryEntries
= 0; IndexOfPageDirectoryEntries
< 512; IndexOfPageDirectoryEntries
++, PageDirectory1GEntry
++, PageAddress
+= SIZE_1GB
) {
877 if (ToSplitPageTable (PageAddress
, SIZE_1GB
, StackBase
, StackSize
, GhcbBase
, GhcbSize
)) {
878 Split1GPageTo2M (PageAddress
, (UINT64
*)PageDirectory1GEntry
, StackBase
, StackSize
, GhcbBase
, GhcbSize
);
881 // Fill in the Page Directory entries
883 PageDirectory1GEntry
->Uint64
= (UINT64
)PageAddress
| AddressEncMask
;
884 PageDirectory1GEntry
->Bits
.ReadWrite
= 1;
885 PageDirectory1GEntry
->Bits
.Present
= 1;
886 PageDirectory1GEntry
->Bits
.MustBe1
= 1;
890 for ( IndexOfPdpEntries
= 0
891 ; IndexOfPdpEntries
< (NumberOfPml4EntriesNeeded
== 1 ? NumberOfPdpEntriesNeeded
: 512)
892 ; IndexOfPdpEntries
++, PageDirectoryPointerEntry
++)
895 // Each Directory Pointer entries points to a page of Page Directory entires.
896 // So allocate space for them and fill them in in the IndexOfPageDirectoryEntries loop.
898 PageDirectoryEntry
= (VOID
*)BigPageAddress
;
899 BigPageAddress
+= SIZE_4KB
;
902 // Fill in a Page Directory Pointer Entries
904 PageDirectoryPointerEntry
->Uint64
= (UINT64
)(UINTN
)PageDirectoryEntry
| AddressEncMask
;
905 PageDirectoryPointerEntry
->Bits
.ReadWrite
= 1;
906 PageDirectoryPointerEntry
->Bits
.Present
= 1;
908 for (IndexOfPageDirectoryEntries
= 0; IndexOfPageDirectoryEntries
< 512; IndexOfPageDirectoryEntries
++, PageDirectoryEntry
++, PageAddress
+= SIZE_2MB
) {
909 if (ToSplitPageTable (PageAddress
, SIZE_2MB
, StackBase
, StackSize
, GhcbBase
, GhcbSize
)) {
911 // Need to split this 2M page that covers NULL or stack range.
913 Split2MPageTo4K (PageAddress
, (UINT64
*)PageDirectoryEntry
, StackBase
, StackSize
, GhcbBase
, GhcbSize
);
916 // Fill in the Page Directory entries
918 PageDirectoryEntry
->Uint64
= (UINT64
)PageAddress
| AddressEncMask
;
919 PageDirectoryEntry
->Bits
.ReadWrite
= 1;
920 PageDirectoryEntry
->Bits
.Present
= 1;
921 PageDirectoryEntry
->Bits
.MustBe1
= 1;
927 // Fill with null entry for unused PDPTE
929 ZeroMem (PageDirectoryPointerEntry
, (512 - IndexOfPdpEntries
) * sizeof (PAGE_MAP_AND_DIRECTORY_POINTER
));
934 // For the PML4 entries we are not using fill in a null entry.
936 ZeroMem (PageMapLevel4Entry
, (512 - IndexOfPml4Entries
) * sizeof (PAGE_MAP_AND_DIRECTORY_POINTER
));
939 if (Page5LevelSupport
) {
940 Cr4
.UintN
= AsmReadCr4 ();
942 AsmWriteCr4 (Cr4
.UintN
);
944 // For the PML5 entries we are not using fill in a null entry.
946 ZeroMem (PageMapLevel5Entry
, (512 - IndexOfPml5Entries
) * sizeof (PAGE_MAP_AND_DIRECTORY_POINTER
));
950 // Protect the page table by marking the memory used for page table to be
953 EnablePageTableProtection ((UINTN
)PageMap
, TRUE
);
956 // Set IA32_EFER.NXE if necessary.
958 if (IsEnableNonExecNeeded ()) {
959 EnableExecuteDisableBit ();
962 return (UINTN
)PageMap
;