2 Page Fault (#PF) handler for X64 processors
4 Copyright (c) 2009 - 2019, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
11 #include "PiSmmCpuDxeSmm.h"
13 #define PAGE_TABLE_PAGES 8
14 #define ACC_MAX_BIT BIT3
16 LIST_ENTRY mPagePool
= INITIALIZE_LIST_HEAD_VARIABLE (mPagePool
);
17 BOOLEAN m1GPageTableSupport
= FALSE
;
18 BOOLEAN mCpuSmmStaticPageTable
;
19 BOOLEAN m5LevelPagingSupport
;
20 X86_ASSEMBLY_PATCH_LABEL gPatch5LevelPagingSupport
;
41 Check if 1-GByte pages is supported by processor or not.
43 @retval TRUE 1-GByte pages is supported.
44 @retval FALSE 1-GByte pages is not supported.
55 AsmCpuid (0x80000000, &RegEax
, NULL
, NULL
, NULL
);
56 if (RegEax
>= 0x80000001) {
57 AsmCpuid (0x80000001, NULL
, NULL
, NULL
, &RegEdx
);
58 if ((RegEdx
& BIT26
) != 0) {
66 Check if 5-level paging is supported by processor or not.
68 @retval TRUE 5-level paging is supported.
69 @retval FALSE 5-level paging is not supported.
73 Is5LevelPagingSupport (
77 CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS_ECX EcxFlags
;
80 CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS
,
81 CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS_SUB_LEAF_INFO
,
87 return (BOOLEAN
) (EcxFlags
.Bits
.FiveLevelPage
!= 0);
91 Set sub-entries number in entry.
93 @param[in, out] Entry Pointer to entry
94 @param[in] SubEntryNum Sub-entries number based on 0:
95 0 means there is 1 sub-entry under this entry
96 0x1ff means there is 512 sub-entries under this entry
101 IN OUT UINT64
*Entry
,
102 IN UINT64 SubEntryNum
106 // Sub-entries number is saved in BIT52 to BIT60 (reserved field) in Entry
108 *Entry
= BitFieldWrite64 (*Entry
, 52, 60, SubEntryNum
);
112 Return sub-entries number in entry.
114 @param[in] Entry Pointer to entry
116 @return Sub-entries number based on 0:
117 0 means there is 1 sub-entry under this entry
118 0x1ff means there is 512 sub-entries under this entry
126 // Sub-entries number is saved in BIT52 to BIT60 (reserved field) in Entry
128 return BitFieldRead64 (*Entry
, 52, 60);
132 Calculate the maximum support address.
134 @return the maximum support address.
137 CalculateMaximumSupportAddress (
142 UINT8 PhysicalAddressBits
;
146 // Get physical address bits supported.
148 Hob
= GetFirstHob (EFI_HOB_TYPE_CPU
);
150 PhysicalAddressBits
= ((EFI_HOB_CPU
*) Hob
)->SizeOfMemorySpace
;
152 AsmCpuid (0x80000000, &RegEax
, NULL
, NULL
, NULL
);
153 if (RegEax
>= 0x80000008) {
154 AsmCpuid (0x80000008, &RegEax
, NULL
, NULL
, NULL
);
155 PhysicalAddressBits
= (UINT8
) RegEax
;
157 PhysicalAddressBits
= 36;
160 return PhysicalAddressBits
;
164 Set static page table.
166 @param[in] PageTable Address of page table.
174 UINTN NumberOfPml5EntriesNeeded
;
175 UINTN NumberOfPml4EntriesNeeded
;
176 UINTN NumberOfPdpEntriesNeeded
;
177 UINTN IndexOfPml5Entries
;
178 UINTN IndexOfPml4Entries
;
179 UINTN IndexOfPdpEntries
;
180 UINTN IndexOfPageDirectoryEntries
;
181 UINT64
*PageMapLevel5Entry
;
182 UINT64
*PageMapLevel4Entry
;
184 UINT64
*PageDirectoryPointerEntry
;
185 UINT64
*PageDirectory1GEntry
;
186 UINT64
*PageDirectoryEntry
;
189 // IA-32e paging translates 48-bit linear addresses to 52-bit physical addresses
190 // when 5-Level Paging is disabled.
192 ASSERT (mPhysicalAddressBits
<= 52);
193 if (!m5LevelPagingSupport
&& mPhysicalAddressBits
> 48) {
194 mPhysicalAddressBits
= 48;
197 NumberOfPml5EntriesNeeded
= 1;
198 if (mPhysicalAddressBits
> 48) {
199 NumberOfPml5EntriesNeeded
= (UINTN
) LShiftU64 (1, mPhysicalAddressBits
- 48);
200 mPhysicalAddressBits
= 48;
203 NumberOfPml4EntriesNeeded
= 1;
204 if (mPhysicalAddressBits
> 39) {
205 NumberOfPml4EntriesNeeded
= (UINTN
) LShiftU64 (1, mPhysicalAddressBits
- 39);
206 mPhysicalAddressBits
= 39;
209 NumberOfPdpEntriesNeeded
= 1;
210 ASSERT (mPhysicalAddressBits
> 30);
211 NumberOfPdpEntriesNeeded
= (UINTN
) LShiftU64 (1, mPhysicalAddressBits
- 30);
214 // By architecture only one PageMapLevel4 exists - so lets allocate storage for it.
216 PageMap
= (VOID
*) PageTable
;
218 PageMapLevel4Entry
= PageMap
;
219 PageMapLevel5Entry
= NULL
;
220 if (m5LevelPagingSupport
) {
222 // By architecture only one PageMapLevel5 exists - so lets allocate storage for it.
224 PageMapLevel5Entry
= PageMap
;
228 for ( IndexOfPml5Entries
= 0
229 ; IndexOfPml5Entries
< NumberOfPml5EntriesNeeded
230 ; IndexOfPml5Entries
++, PageMapLevel5Entry
++) {
232 // Each PML5 entry points to a page of PML4 entires.
233 // So lets allocate space for them and fill them in in the IndexOfPml4Entries loop.
234 // When 5-Level Paging is disabled, below allocation happens only once.
236 if (m5LevelPagingSupport
) {
237 PageMapLevel4Entry
= (UINT64
*) ((*PageMapLevel5Entry
) & ~mAddressEncMask
& gPhyMask
);
238 if (PageMapLevel4Entry
== NULL
) {
239 PageMapLevel4Entry
= AllocatePageTableMemory (1);
240 ASSERT(PageMapLevel4Entry
!= NULL
);
241 ZeroMem (PageMapLevel4Entry
, EFI_PAGES_TO_SIZE(1));
243 *PageMapLevel5Entry
= (UINT64
)(UINTN
)PageMapLevel4Entry
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
247 for (IndexOfPml4Entries
= 0; IndexOfPml4Entries
< (NumberOfPml5EntriesNeeded
== 1 ? NumberOfPml4EntriesNeeded
: 512); IndexOfPml4Entries
++, PageMapLevel4Entry
++) {
249 // Each PML4 entry points to a page of Page Directory Pointer entries.
251 PageDirectoryPointerEntry
= (UINT64
*) ((*PageMapLevel4Entry
) & ~mAddressEncMask
& gPhyMask
);
252 if (PageDirectoryPointerEntry
== NULL
) {
253 PageDirectoryPointerEntry
= AllocatePageTableMemory (1);
254 ASSERT(PageDirectoryPointerEntry
!= NULL
);
255 ZeroMem (PageDirectoryPointerEntry
, EFI_PAGES_TO_SIZE(1));
257 *PageMapLevel4Entry
= (UINT64
)(UINTN
)PageDirectoryPointerEntry
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
260 if (m1GPageTableSupport
) {
261 PageDirectory1GEntry
= PageDirectoryPointerEntry
;
262 for (IndexOfPageDirectoryEntries
= 0; IndexOfPageDirectoryEntries
< 512; IndexOfPageDirectoryEntries
++, PageDirectory1GEntry
++, PageAddress
+= SIZE_1GB
) {
263 if (IndexOfPml4Entries
== 0 && IndexOfPageDirectoryEntries
< 4) {
265 // Skip the < 4G entries
270 // Fill in the Page Directory entries
272 *PageDirectory1GEntry
= PageAddress
| mAddressEncMask
| IA32_PG_PS
| PAGE_ATTRIBUTE_BITS
;
275 PageAddress
= BASE_4GB
;
276 for (IndexOfPdpEntries
= 0; IndexOfPdpEntries
< (NumberOfPml4EntriesNeeded
== 1 ? NumberOfPdpEntriesNeeded
: 512); IndexOfPdpEntries
++, PageDirectoryPointerEntry
++) {
277 if (IndexOfPml4Entries
== 0 && IndexOfPdpEntries
< 4) {
279 // Skip the < 4G entries
284 // Each Directory Pointer entries points to a page of Page Directory entires.
285 // So allocate space for them and fill them in in the IndexOfPageDirectoryEntries loop.
287 PageDirectoryEntry
= (UINT64
*) ((*PageDirectoryPointerEntry
) & ~mAddressEncMask
& gPhyMask
);
288 if (PageDirectoryEntry
== NULL
) {
289 PageDirectoryEntry
= AllocatePageTableMemory (1);
290 ASSERT(PageDirectoryEntry
!= NULL
);
291 ZeroMem (PageDirectoryEntry
, EFI_PAGES_TO_SIZE(1));
294 // Fill in a Page Directory Pointer Entries
296 *PageDirectoryPointerEntry
= (UINT64
)(UINTN
)PageDirectoryEntry
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
299 for (IndexOfPageDirectoryEntries
= 0; IndexOfPageDirectoryEntries
< 512; IndexOfPageDirectoryEntries
++, PageDirectoryEntry
++, PageAddress
+= SIZE_2MB
) {
301 // Fill in the Page Directory entries
303 *PageDirectoryEntry
= PageAddress
| mAddressEncMask
| IA32_PG_PS
| PAGE_ATTRIBUTE_BITS
;
312 Create PageTable for SMM use.
314 @return The address of PML4 (to set CR3).
322 EFI_PHYSICAL_ADDRESS Pages
;
324 LIST_ENTRY
*FreePage
;
326 UINTN PageFaultHandlerHookAddress
;
327 IA32_IDT_GATE_DESCRIPTOR
*IdtEntry
;
333 // Initialize spin lock
335 InitializeSpinLock (mPFLock
);
337 mCpuSmmStaticPageTable
= PcdGetBool (PcdCpuSmmStaticPageTable
);
338 m1GPageTableSupport
= Is1GPageSupport ();
339 m5LevelPagingSupport
= Is5LevelPagingSupport ();
340 mPhysicalAddressBits
= CalculateMaximumSupportAddress ();
341 PatchInstructionX86 (gPatch5LevelPagingSupport
, m5LevelPagingSupport
, 1);
342 DEBUG ((DEBUG_INFO
, "5LevelPaging Support - %d\n", m5LevelPagingSupport
));
343 DEBUG ((DEBUG_INFO
, "1GPageTable Support - %d\n", m1GPageTableSupport
));
344 DEBUG ((DEBUG_INFO
, "PcdCpuSmmStaticPageTable - %d\n", mCpuSmmStaticPageTable
));
345 DEBUG ((DEBUG_INFO
, "PhysicalAddressBits - %d\n", mPhysicalAddressBits
));
347 // Generate PAE page table for the first 4GB memory space
349 Pages
= Gen4GPageTable (FALSE
);
352 // Set IA32_PG_PMNT bit to mask this entry
354 PTEntry
= (UINT64
*)(UINTN
)Pages
;
355 for (Index
= 0; Index
< 4; Index
++) {
356 PTEntry
[Index
] |= IA32_PG_PMNT
;
360 // Fill Page-Table-Level4 (PML4) entry
362 Pml4Entry
= (UINT64
*)AllocatePageTableMemory (1);
363 ASSERT (Pml4Entry
!= NULL
);
364 *Pml4Entry
= Pages
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
365 ZeroMem (Pml4Entry
+ 1, EFI_PAGE_SIZE
- sizeof (*Pml4Entry
));
368 // Set sub-entries number
370 SetSubEntriesNum (Pml4Entry
, 3);
373 if (m5LevelPagingSupport
) {
377 Pml5Entry
= (UINT64
*)AllocatePageTableMemory (1);
378 *Pml5Entry
= (UINTN
) Pml4Entry
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
379 ZeroMem (Pml5Entry
+ 1, EFI_PAGE_SIZE
- sizeof (*Pml5Entry
));
381 // Set sub-entries number
383 SetSubEntriesNum (Pml5Entry
, 1);
387 if (mCpuSmmStaticPageTable
) {
388 SetStaticPageTable ((UINTN
)PTEntry
);
391 // Add pages to page pool
393 FreePage
= (LIST_ENTRY
*)AllocatePageTableMemory (PAGE_TABLE_PAGES
);
394 ASSERT (FreePage
!= NULL
);
395 for (Index
= 0; Index
< PAGE_TABLE_PAGES
; Index
++) {
396 InsertTailList (&mPagePool
, FreePage
);
397 FreePage
+= EFI_PAGE_SIZE
/ sizeof (*FreePage
);
401 if (FeaturePcdGet (PcdCpuSmmProfileEnable
) ||
402 HEAP_GUARD_NONSTOP_MODE
||
403 NULL_DETECTION_NONSTOP_MODE
) {
405 // Set own Page Fault entry instead of the default one, because SMM Profile
406 // feature depends on IRET instruction to do Single Step
408 PageFaultHandlerHookAddress
= (UINTN
)PageFaultIdtHandlerSmmProfile
;
409 IdtEntry
= (IA32_IDT_GATE_DESCRIPTOR
*) gcSmiIdtr
.Base
;
410 IdtEntry
+= EXCEPT_IA32_PAGE_FAULT
;
411 IdtEntry
->Bits
.OffsetLow
= (UINT16
)PageFaultHandlerHookAddress
;
412 IdtEntry
->Bits
.Reserved_0
= 0;
413 IdtEntry
->Bits
.GateType
= IA32_IDT_GATE_TYPE_INTERRUPT_32
;
414 IdtEntry
->Bits
.OffsetHigh
= (UINT16
)(PageFaultHandlerHookAddress
>> 16);
415 IdtEntry
->Bits
.OffsetUpper
= (UINT32
)(PageFaultHandlerHookAddress
>> 32);
416 IdtEntry
->Bits
.Reserved_1
= 0;
419 // Register Smm Page Fault Handler
421 Status
= SmmRegisterExceptionHandler (&mSmmCpuService
, EXCEPT_IA32_PAGE_FAULT
, SmiPFHandler
);
422 ASSERT_EFI_ERROR (Status
);
426 // Additional SMM IDT initialization for SMM stack guard
428 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
429 InitializeIDTSmmStackGuard ();
433 // Return the address of PML4/PML5 (to set CR3)
435 return (UINT32
)(UINTN
)PTEntry
;
439 Set access record in entry.
441 @param[in, out] Entry Pointer to entry
442 @param[in] Acc Access record value
447 IN OUT UINT64
*Entry
,
452 // Access record is saved in BIT9 to BIT11 (reserved field) in Entry
454 *Entry
= BitFieldWrite64 (*Entry
, 9, 11, Acc
);
458 Return access record in entry.
460 @param[in] Entry Pointer to entry
462 @return Access record value.
471 // Access record is saved in BIT9 to BIT11 (reserved field) in Entry
473 return BitFieldRead64 (*Entry
, 9, 11);
477 Return and update the access record in entry.
479 @param[in, out] Entry Pointer to entry
481 @return Access record value.
491 Acc
= GetAccNum (Entry
);
492 if ((*Entry
& IA32_PG_A
) != 0) {
494 // If this entry has been accessed, clear access flag in Entry and update access record
495 // to the initial value 7, adding ACC_MAX_BIT is to make it larger than others
497 *Entry
&= ~(UINT64
)(UINTN
)IA32_PG_A
;
498 SetAccNum (Entry
, 0x7);
499 return (0x7 + ACC_MAX_BIT
);
503 // If the access record is not the smallest value 0, minus 1 and update the access record field
505 SetAccNum (Entry
, Acc
- 1);
512 Reclaim free pages for PageFault handler.
514 Search the whole entries tree to find the leaf entry that has the smallest
515 access record value. Insert the page pointed by this leaf entry into the
516 page pool. And check its upper entries if need to be inserted into the page
540 UINT64 SubEntriesNum
;
543 UINT64
*ReleasePageAddress
;
545 BOOLEAN Enable5LevelPaging
;
556 ReleasePageAddress
= 0;
558 Cr4
.UintN
= AsmReadCr4 ();
559 Enable5LevelPaging
= (BOOLEAN
) (Cr4
.Bits
.LA57
== 1);
560 Pml5
= (UINT64
*)(UINTN
)(AsmReadCr3 () & gPhyMask
);
562 if (!Enable5LevelPaging
) {
564 // Create one fake PML5 entry for 4-Level Paging
565 // so that the page table parsing logic only handles 5-Level page structure.
567 Pml5Entry
= (UINTN
) Pml5
| IA32_PG_P
;
572 // First, find the leaf entry has the smallest access record value
574 for (Pml5Index
= 0; Pml5Index
< (Enable5LevelPaging
? (EFI_PAGE_SIZE
/ sizeof (*Pml4
)) : 1); Pml5Index
++) {
575 if ((Pml5
[Pml5Index
] & IA32_PG_P
) == 0 || (Pml5
[Pml5Index
] & IA32_PG_PMNT
) != 0) {
577 // If the PML5 entry is not present or is masked, skip it
581 Pml4
= (UINT64
*)(UINTN
)(Pml5
[Pml5Index
] & gPhyMask
);
582 for (Pml4Index
= 0; Pml4Index
< EFI_PAGE_SIZE
/ sizeof (*Pml4
); Pml4Index
++) {
583 if ((Pml4
[Pml4Index
] & IA32_PG_P
) == 0 || (Pml4
[Pml4Index
] & IA32_PG_PMNT
) != 0) {
585 // If the PML4 entry is not present or is masked, skip it
589 Pdpt
= (UINT64
*)(UINTN
)(Pml4
[Pml4Index
] & ~mAddressEncMask
& gPhyMask
);
591 for (PdptIndex
= 0; PdptIndex
< EFI_PAGE_SIZE
/ sizeof (*Pdpt
); PdptIndex
++) {
592 if ((Pdpt
[PdptIndex
] & IA32_PG_P
) == 0 || (Pdpt
[PdptIndex
] & IA32_PG_PMNT
) != 0) {
594 // If the PDPT entry is not present or is masked, skip it
596 if ((Pdpt
[PdptIndex
] & IA32_PG_PMNT
) != 0) {
598 // If the PDPT entry is masked, we will ignore checking the PML4 entry
604 if ((Pdpt
[PdptIndex
] & IA32_PG_PS
) == 0) {
606 // It's not 1-GByte pages entry, it should be a PDPT entry,
607 // we will not check PML4 entry more
610 Pdt
= (UINT64
*)(UINTN
)(Pdpt
[PdptIndex
] & ~mAddressEncMask
& gPhyMask
);
612 for (PdtIndex
= 0; PdtIndex
< EFI_PAGE_SIZE
/ sizeof(*Pdt
); PdtIndex
++) {
613 if ((Pdt
[PdtIndex
] & IA32_PG_P
) == 0 || (Pdt
[PdtIndex
] & IA32_PG_PMNT
) != 0) {
615 // If the PD entry is not present or is masked, skip it
617 if ((Pdt
[PdtIndex
] & IA32_PG_PMNT
) != 0) {
619 // If the PD entry is masked, we will not PDPT entry more
625 if ((Pdt
[PdtIndex
] & IA32_PG_PS
) == 0) {
627 // It's not 2 MByte page table entry, it should be PD entry
628 // we will find the entry has the smallest access record value
631 Acc
= GetAndUpdateAccNum (Pdt
+ PdtIndex
);
634 // If the PD entry has the smallest access record value,
635 // save the Page address to be released
642 ReleasePageAddress
= Pdt
+ PdtIndex
;
648 // If this PDPT entry has no PDT entries pointer to 4 KByte pages,
649 // it should only has the entries point to 2 MByte Pages
651 Acc
= GetAndUpdateAccNum (Pdpt
+ PdptIndex
);
654 // If the PDPT entry has the smallest access record value,
655 // save the Page address to be released
662 ReleasePageAddress
= Pdpt
+ PdptIndex
;
669 // If PML4 entry has no the PDPT entry pointer to 2 MByte pages,
670 // it should only has the entries point to 1 GByte Pages
672 Acc
= GetAndUpdateAccNum (Pml4
+ Pml4Index
);
675 // If the PML4 entry has the smallest access record value,
676 // save the Page address to be released
683 ReleasePageAddress
= Pml4
+ Pml4Index
;
689 // Make sure one PML4/PDPT/PD entry is selected
691 ASSERT (MinAcc
!= (UINT64
)-1);
694 // Secondly, insert the page pointed by this entry into page pool and clear this entry
696 InsertTailList (&mPagePool
, (LIST_ENTRY
*)(UINTN
)(*ReleasePageAddress
& ~mAddressEncMask
& gPhyMask
));
697 *ReleasePageAddress
= 0;
700 // Lastly, check this entry's upper entries if need to be inserted into page pool
704 if (MinPdt
!= (UINTN
)-1) {
706 // If 4 KByte Page Table is released, check the PDPT entry
708 Pml4
= (UINT64
*) (UINTN
) (Pml5
[MinPml5
] & gPhyMask
);
709 Pdpt
= (UINT64
*)(UINTN
)(Pml4
[MinPml4
] & ~mAddressEncMask
& gPhyMask
);
710 SubEntriesNum
= GetSubEntriesNum(Pdpt
+ MinPdpt
);
711 if (SubEntriesNum
== 0) {
713 // Release the empty Page Directory table if there was no more 4 KByte Page Table entry
714 // clear the Page directory entry
716 InsertTailList (&mPagePool
, (LIST_ENTRY
*)(UINTN
)(Pdpt
[MinPdpt
] & ~mAddressEncMask
& gPhyMask
));
719 // Go on checking the PML4 table
725 // Update the sub-entries filed in PDPT entry and exit
727 SetSubEntriesNum (Pdpt
+ MinPdpt
, SubEntriesNum
- 1);
730 if (MinPdpt
!= (UINTN
)-1) {
732 // One 2MB Page Table is released or Page Directory table is released, check the PML4 entry
734 SubEntriesNum
= GetSubEntriesNum (Pml4
+ MinPml4
);
735 if (SubEntriesNum
== 0) {
737 // Release the empty PML4 table if there was no more 1G KByte Page Table entry
738 // clear the Page directory entry
740 InsertTailList (&mPagePool
, (LIST_ENTRY
*)(UINTN
)(Pml4
[MinPml4
] & ~mAddressEncMask
& gPhyMask
));
746 // Update the sub-entries filed in PML4 entry and exit
748 SetSubEntriesNum (Pml4
+ MinPml4
, SubEntriesNum
- 1);
752 // PLM4 table has been released before, exit it
759 Allocate free Page for PageFault handler use.
761 @return Page address.
771 if (IsListEmpty (&mPagePool
)) {
773 // If page pool is empty, reclaim the used pages and insert one into page pool
779 // Get one free page and remove it from page pool
781 RetVal
= (UINT64
)(UINTN
)mPagePool
.ForwardLink
;
782 RemoveEntryList (mPagePool
.ForwardLink
);
784 // Clean this page and return
786 ZeroMem ((VOID
*)(UINTN
)RetVal
, EFI_PAGE_SIZE
);
791 Page Fault handler for SMM use.
795 SmiDefaultPFHandler (
800 UINT64
*PageTableTop
;
806 SMM_PAGE_SIZE_TYPE PageSize
;
811 BOOLEAN Enable5LevelPaging
;
815 // Set default SMM page attribute
817 PageSize
= SmmPageSize2M
;
822 PageTableTop
= (UINT64
*)(AsmReadCr3 () & gPhyMask
);
823 PFAddress
= AsmReadCr2 ();
825 Cr4
.UintN
= AsmReadCr4 ();
826 Enable5LevelPaging
= (BOOLEAN
) (Cr4
.Bits
.LA57
!= 0);
828 Status
= GetPlatformPageTableAttribute (PFAddress
, &PageSize
, &NumOfPages
, &PageAttribute
);
830 // If platform not support page table attribute, set default SMM page attribute
832 if (Status
!= EFI_SUCCESS
) {
833 PageSize
= SmmPageSize2M
;
837 if (PageSize
>= MaxSmmPageSizeType
) {
838 PageSize
= SmmPageSize2M
;
840 if (NumOfPages
> 512) {
847 // BIT12 to BIT20 is Page Table index
853 // BIT21 to BIT29 is Page Directory index
856 PageAttribute
|= (UINTN
)IA32_PG_PS
;
859 if (!m1GPageTableSupport
) {
860 DEBUG ((DEBUG_ERROR
, "1-GByte pages is not supported!"));
864 // BIT30 to BIT38 is Page Directory Pointer Table index
867 PageAttribute
|= (UINTN
)IA32_PG_PS
;
874 // If execute-disable is enabled, set NX bit
877 PageAttribute
|= IA32_PG_NX
;
880 for (Index
= 0; Index
< NumOfPages
; Index
++) {
881 PageTable
= PageTableTop
;
883 for (StartBit
= Enable5LevelPaging
? 48 : 39; StartBit
> EndBit
; StartBit
-= 9) {
884 PTIndex
= BitFieldRead64 (PFAddress
, StartBit
, StartBit
+ 8);
885 if ((PageTable
[PTIndex
] & IA32_PG_P
) == 0) {
887 // If the entry is not present, allocate one page from page pool for it
889 PageTable
[PTIndex
] = AllocPage () | mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
892 // Save the upper entry address
894 UpperEntry
= PageTable
+ PTIndex
;
897 // BIT9 to BIT11 of entry is used to save access record,
898 // initialize value is 7
900 PageTable
[PTIndex
] |= (UINT64
)IA32_PG_A
;
901 SetAccNum (PageTable
+ PTIndex
, 7);
902 PageTable
= (UINT64
*)(UINTN
)(PageTable
[PTIndex
] & ~mAddressEncMask
& gPhyMask
);
905 PTIndex
= BitFieldRead64 (PFAddress
, StartBit
, StartBit
+ 8);
906 if ((PageTable
[PTIndex
] & IA32_PG_P
) != 0) {
908 // Check if the entry has already existed, this issue may occur when the different
909 // size page entries created under the same entry
911 DEBUG ((DEBUG_ERROR
, "PageTable = %lx, PTIndex = %x, PageTable[PTIndex] = %lx\n", PageTable
, PTIndex
, PageTable
[PTIndex
]));
912 DEBUG ((DEBUG_ERROR
, "New page table overlapped with old page table!\n"));
916 // Fill the new entry
918 PageTable
[PTIndex
] = ((PFAddress
| mAddressEncMask
) & gPhyMask
& ~((1ull << EndBit
) - 1)) |
919 PageAttribute
| IA32_PG_A
| PAGE_ATTRIBUTE_BITS
;
920 if (UpperEntry
!= NULL
) {
921 SetSubEntriesNum (UpperEntry
, GetSubEntriesNum (UpperEntry
) + 1);
924 // Get the next page address if we need to create more page tables
926 PFAddress
+= (1ull << EndBit
);
931 ThePage Fault handler wrapper for SMM use.
933 @param InterruptType Defines the type of interrupt or exception that
934 occurred on the processor.This parameter is processor architecture specific.
935 @param SystemContext A pointer to the processor context when
936 the interrupt occurred on the processor.
941 IN EFI_EXCEPTION_TYPE InterruptType
,
942 IN EFI_SYSTEM_CONTEXT SystemContext
946 UINTN GuardPageAddress
;
949 ASSERT (InterruptType
== EXCEPT_IA32_PAGE_FAULT
);
951 AcquireSpinLock (mPFLock
);
953 PFAddress
= AsmReadCr2 ();
955 if (mCpuSmmStaticPageTable
&& (PFAddress
>= LShiftU64 (1, (mPhysicalAddressBits
- 1)))) {
956 DumpCpuContext (InterruptType
, SystemContext
);
957 DEBUG ((DEBUG_ERROR
, "Do not support address 0x%lx by processor!\n", PFAddress
));
963 // If a page fault occurs in SMRAM range, it might be in a SMM stack guard page,
964 // or SMM page protection violation.
966 if ((PFAddress
>= mCpuHotPlugData
.SmrrBase
) &&
967 (PFAddress
< (mCpuHotPlugData
.SmrrBase
+ mCpuHotPlugData
.SmrrSize
))) {
968 DumpCpuContext (InterruptType
, SystemContext
);
969 CpuIndex
= GetCpuIndex ();
970 GuardPageAddress
= (mSmmStackArrayBase
+ EFI_PAGE_SIZE
+ CpuIndex
* mSmmStackSize
);
971 if ((FeaturePcdGet (PcdCpuSmmStackGuard
)) &&
972 (PFAddress
>= GuardPageAddress
) &&
973 (PFAddress
< (GuardPageAddress
+ EFI_PAGE_SIZE
))) {
974 DEBUG ((DEBUG_ERROR
, "SMM stack overflow!\n"));
976 if ((SystemContext
.SystemContextX64
->ExceptionData
& IA32_PF_EC_ID
) != 0) {
977 DEBUG ((DEBUG_ERROR
, "SMM exception at execution (0x%lx)\n", PFAddress
));
979 DumpModuleInfoByIp (*(UINTN
*)(UINTN
)SystemContext
.SystemContextX64
->Rsp
);
982 DEBUG ((DEBUG_ERROR
, "SMM exception at access (0x%lx)\n", PFAddress
));
984 DumpModuleInfoByIp ((UINTN
)SystemContext
.SystemContextX64
->Rip
);
988 if (HEAP_GUARD_NONSTOP_MODE
) {
989 GuardPagePFHandler (SystemContext
.SystemContextX64
->ExceptionData
);
998 // If a page fault occurs in non-SMRAM range.
1000 if ((PFAddress
< mCpuHotPlugData
.SmrrBase
) ||
1001 (PFAddress
>= mCpuHotPlugData
.SmrrBase
+ mCpuHotPlugData
.SmrrSize
)) {
1002 if ((SystemContext
.SystemContextX64
->ExceptionData
& IA32_PF_EC_ID
) != 0) {
1003 DumpCpuContext (InterruptType
, SystemContext
);
1004 DEBUG ((DEBUG_ERROR
, "Code executed on IP(0x%lx) out of SMM range after SMM is locked!\n", PFAddress
));
1006 DumpModuleInfoByIp (*(UINTN
*)(UINTN
)SystemContext
.SystemContextX64
->Rsp
);
1013 // If NULL pointer was just accessed
1015 if ((PcdGet8 (PcdNullPointerDetectionPropertyMask
) & BIT1
) != 0 &&
1016 (PFAddress
< EFI_PAGE_SIZE
)) {
1017 DumpCpuContext (InterruptType
, SystemContext
);
1018 DEBUG ((DEBUG_ERROR
, "!!! NULL pointer access !!!\n"));
1020 DumpModuleInfoByIp ((UINTN
)SystemContext
.SystemContextX64
->Rip
);
1023 if (NULL_DETECTION_NONSTOP_MODE
) {
1024 GuardPagePFHandler (SystemContext
.SystemContextX64
->ExceptionData
);
1032 if (mCpuSmmStaticPageTable
&& IsSmmCommBufferForbiddenAddress (PFAddress
)) {
1033 DumpCpuContext (InterruptType
, SystemContext
);
1034 DEBUG ((DEBUG_ERROR
, "Access SMM communication forbidden address (0x%lx)!\n", PFAddress
));
1036 DumpModuleInfoByIp ((UINTN
)SystemContext
.SystemContextX64
->Rip
);
1043 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1044 SmmProfilePFHandler (
1045 SystemContext
.SystemContextX64
->Rip
,
1046 SystemContext
.SystemContextX64
->ExceptionData
1049 SmiDefaultPFHandler ();
1053 ReleaseSpinLock (mPFLock
);
1057 This function sets memory attribute for page table.
1060 SetPageTableAttributes (
1068 UINT64
*L1PageTable
;
1069 UINT64
*L2PageTable
;
1070 UINT64
*L3PageTable
;
1071 UINT64
*L4PageTable
;
1072 UINT64
*L5PageTable
;
1074 BOOLEAN PageTableSplitted
;
1077 BOOLEAN Enable5LevelPaging
;
1079 Cr4
.UintN
= AsmReadCr4 ();
1080 Enable5LevelPaging
= (BOOLEAN
) (Cr4
.Bits
.LA57
== 1);
1084 // - no static page table; or
1085 // - SMM heap guard feature enabled; or
1086 // BIT2: SMM page guard enabled
1087 // BIT3: SMM pool guard enabled
1088 // - SMM profile feature enabled
1090 if (!mCpuSmmStaticPageTable
||
1091 ((PcdGet8 (PcdHeapGuardPropertyMask
) & (BIT3
| BIT2
)) != 0) ||
1092 FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1094 // Static paging and heap guard could not be enabled at the same time.
1096 ASSERT (!(mCpuSmmStaticPageTable
&&
1097 (PcdGet8 (PcdHeapGuardPropertyMask
) & (BIT3
| BIT2
)) != 0));
1100 // Static paging and SMM profile could not be enabled at the same time.
1102 ASSERT (!(mCpuSmmStaticPageTable
&& FeaturePcdGet (PcdCpuSmmProfileEnable
)));
1106 DEBUG ((DEBUG_INFO
, "SetPageTableAttributes\n"));
1109 // Disable write protection, because we need mark page table to be write protected.
1110 // We need *write* page table memory, to mark itself to be *read only*.
1112 CetEnabled
= ((AsmReadCr4() & CR4_CET_ENABLE
) != 0) ? TRUE
: FALSE
;
1115 // CET must be disabled if WP is disabled.
1119 AsmWriteCr0 (AsmReadCr0() & ~CR0_WP
);
1122 DEBUG ((DEBUG_INFO
, "Start...\n"));
1123 PageTableSplitted
= FALSE
;
1125 if (Enable5LevelPaging
) {
1126 L5PageTable
= (UINT64
*)GetPageTableBase ();
1127 SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS
)(UINTN
)L5PageTable
, SIZE_4KB
, EFI_MEMORY_RO
, &IsSplitted
);
1128 PageTableSplitted
= (PageTableSplitted
|| IsSplitted
);
1131 for (Index5
= 0; Index5
< (Enable5LevelPaging
? SIZE_4KB
/sizeof(UINT64
) : 1); Index5
++) {
1132 if (Enable5LevelPaging
) {
1133 L4PageTable
= (UINT64
*)(UINTN
)(L5PageTable
[Index5
] & ~mAddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
1134 if (L4PageTable
== NULL
) {
1138 L4PageTable
= (UINT64
*)GetPageTableBase ();
1140 SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS
)(UINTN
)L4PageTable
, SIZE_4KB
, EFI_MEMORY_RO
, &IsSplitted
);
1141 PageTableSplitted
= (PageTableSplitted
|| IsSplitted
);
1143 for (Index4
= 0; Index4
< SIZE_4KB
/sizeof(UINT64
); Index4
++) {
1144 L3PageTable
= (UINT64
*)(UINTN
)(L4PageTable
[Index4
] & ~mAddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
1145 if (L3PageTable
== NULL
) {
1149 SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS
)(UINTN
)L3PageTable
, SIZE_4KB
, EFI_MEMORY_RO
, &IsSplitted
);
1150 PageTableSplitted
= (PageTableSplitted
|| IsSplitted
);
1152 for (Index3
= 0; Index3
< SIZE_4KB
/sizeof(UINT64
); Index3
++) {
1153 if ((L3PageTable
[Index3
] & IA32_PG_PS
) != 0) {
1157 L2PageTable
= (UINT64
*)(UINTN
)(L3PageTable
[Index3
] & ~mAddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
1158 if (L2PageTable
== NULL
) {
1162 SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS
)(UINTN
)L2PageTable
, SIZE_4KB
, EFI_MEMORY_RO
, &IsSplitted
);
1163 PageTableSplitted
= (PageTableSplitted
|| IsSplitted
);
1165 for (Index2
= 0; Index2
< SIZE_4KB
/sizeof(UINT64
); Index2
++) {
1166 if ((L2PageTable
[Index2
] & IA32_PG_PS
) != 0) {
1170 L1PageTable
= (UINT64
*)(UINTN
)(L2PageTable
[Index2
] & ~mAddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
1171 if (L1PageTable
== NULL
) {
1174 SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS
)(UINTN
)L1PageTable
, SIZE_4KB
, EFI_MEMORY_RO
, &IsSplitted
);
1175 PageTableSplitted
= (PageTableSplitted
|| IsSplitted
);
1180 } while (PageTableSplitted
);
1183 // Enable write protection, after page table updated.
1185 AsmWriteCr0 (AsmReadCr0() | CR0_WP
);
1197 This function reads CR2 register when on-demand paging is enabled.
1199 @param[out] *Cr2 Pointer to variable to hold CR2 register value.
1206 if (!mCpuSmmStaticPageTable
) {
1207 *Cr2
= AsmReadCr2 ();
1212 This function restores CR2 register when on-demand paging is enabled.
1214 @param[in] Cr2 Value to write into CR2 register.
1221 if (!mCpuSmmStaticPageTable
) {