2 Page Fault (#PF) handler for X64 processors
4 Copyright (c) 2009 - 2019, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
11 #include "PiSmmCpuDxeSmm.h"
13 #define PAGE_TABLE_PAGES 8
14 #define ACC_MAX_BIT BIT3
16 LIST_ENTRY mPagePool
= INITIALIZE_LIST_HEAD_VARIABLE (mPagePool
);
17 BOOLEAN m1GPageTableSupport
= FALSE
;
18 BOOLEAN mCpuSmmRestrictedMemoryAccess
;
19 BOOLEAN m5LevelPagingNeeded
;
20 X86_ASSEMBLY_PATCH_LABEL gPatch5LevelPagingNeeded
;
41 Check if 1-GByte pages is supported by processor or not.
43 @retval TRUE 1-GByte pages is supported.
44 @retval FALSE 1-GByte pages is not supported.
55 AsmCpuid (0x80000000, &RegEax
, NULL
, NULL
, NULL
);
56 if (RegEax
>= 0x80000001) {
57 AsmCpuid (0x80000001, NULL
, NULL
, NULL
, &RegEdx
);
58 if ((RegEdx
& BIT26
) != 0) {
66 The routine returns TRUE when CPU supports it (CPUID[7,0].ECX.BIT[16] is set) and
67 the max physical address bits is bigger than 48. Because 4-level paging can support
68 to address physical address up to 2^48 - 1, there is no need to enable 5-level paging
69 with max physical address bits <= 48.
71 @retval TRUE 5-level paging enabling is needed.
72 @retval FALSE 5-level paging enabling is not needed.
75 Is5LevelPagingNeeded (
79 CPUID_VIR_PHY_ADDRESS_SIZE_EAX VirPhyAddressSize
;
80 CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS_ECX ExtFeatureEcx
;
81 UINT32 MaxExtendedFunctionId
;
83 AsmCpuid (CPUID_EXTENDED_FUNCTION
, &MaxExtendedFunctionId
, NULL
, NULL
, NULL
);
84 if (MaxExtendedFunctionId
>= CPUID_VIR_PHY_ADDRESS_SIZE
) {
85 AsmCpuid (CPUID_VIR_PHY_ADDRESS_SIZE
, &VirPhyAddressSize
.Uint32
, NULL
, NULL
, NULL
);
87 VirPhyAddressSize
.Bits
.PhysicalAddressBits
= 36;
90 CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS
,
91 CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS_SUB_LEAF_INFO
,
92 NULL
, NULL
, &ExtFeatureEcx
.Uint32
, NULL
95 DEBUG_INFO
, "PhysicalAddressBits = %d, 5LPageTable = %d.\n",
96 VirPhyAddressSize
.Bits
.PhysicalAddressBits
, ExtFeatureEcx
.Bits
.FiveLevelPage
99 if (VirPhyAddressSize
.Bits
.PhysicalAddressBits
> 4 * 9 + 12) {
100 ASSERT (ExtFeatureEcx
.Bits
.FiveLevelPage
== 1);
108 Set sub-entries number in entry.
110 @param[in, out] Entry Pointer to entry
111 @param[in] SubEntryNum Sub-entries number based on 0:
112 0 means there is 1 sub-entry under this entry
113 0x1ff means there is 512 sub-entries under this entry
118 IN OUT UINT64
*Entry
,
119 IN UINT64 SubEntryNum
123 // Sub-entries number is saved in BIT52 to BIT60 (reserved field) in Entry
125 *Entry
= BitFieldWrite64 (*Entry
, 52, 60, SubEntryNum
);
129 Return sub-entries number in entry.
131 @param[in] Entry Pointer to entry
133 @return Sub-entries number based on 0:
134 0 means there is 1 sub-entry under this entry
135 0x1ff means there is 512 sub-entries under this entry
143 // Sub-entries number is saved in BIT52 to BIT60 (reserved field) in Entry
145 return BitFieldRead64 (*Entry
, 52, 60);
149 Calculate the maximum support address.
151 @return the maximum support address.
154 CalculateMaximumSupportAddress (
159 UINT8 PhysicalAddressBits
;
163 // Get physical address bits supported.
165 Hob
= GetFirstHob (EFI_HOB_TYPE_CPU
);
167 PhysicalAddressBits
= ((EFI_HOB_CPU
*) Hob
)->SizeOfMemorySpace
;
169 AsmCpuid (0x80000000, &RegEax
, NULL
, NULL
, NULL
);
170 if (RegEax
>= 0x80000008) {
171 AsmCpuid (0x80000008, &RegEax
, NULL
, NULL
, NULL
);
172 PhysicalAddressBits
= (UINT8
) RegEax
;
174 PhysicalAddressBits
= 36;
177 return PhysicalAddressBits
;
181 Set static page table.
183 @param[in] PageTable Address of page table.
191 UINTN NumberOfPml5EntriesNeeded
;
192 UINTN NumberOfPml4EntriesNeeded
;
193 UINTN NumberOfPdpEntriesNeeded
;
194 UINTN IndexOfPml5Entries
;
195 UINTN IndexOfPml4Entries
;
196 UINTN IndexOfPdpEntries
;
197 UINTN IndexOfPageDirectoryEntries
;
198 UINT64
*PageMapLevel5Entry
;
199 UINT64
*PageMapLevel4Entry
;
201 UINT64
*PageDirectoryPointerEntry
;
202 UINT64
*PageDirectory1GEntry
;
203 UINT64
*PageDirectoryEntry
;
206 // IA-32e paging translates 48-bit linear addresses to 52-bit physical addresses
207 // when 5-Level Paging is disabled.
209 ASSERT (mPhysicalAddressBits
<= 52);
210 if (!m5LevelPagingNeeded
&& mPhysicalAddressBits
> 48) {
211 mPhysicalAddressBits
= 48;
214 NumberOfPml5EntriesNeeded
= 1;
215 if (mPhysicalAddressBits
> 48) {
216 NumberOfPml5EntriesNeeded
= (UINTN
) LShiftU64 (1, mPhysicalAddressBits
- 48);
217 mPhysicalAddressBits
= 48;
220 NumberOfPml4EntriesNeeded
= 1;
221 if (mPhysicalAddressBits
> 39) {
222 NumberOfPml4EntriesNeeded
= (UINTN
) LShiftU64 (1, mPhysicalAddressBits
- 39);
223 mPhysicalAddressBits
= 39;
226 NumberOfPdpEntriesNeeded
= 1;
227 ASSERT (mPhysicalAddressBits
> 30);
228 NumberOfPdpEntriesNeeded
= (UINTN
) LShiftU64 (1, mPhysicalAddressBits
- 30);
231 // By architecture only one PageMapLevel4 exists - so lets allocate storage for it.
233 PageMap
= (VOID
*) PageTable
;
235 PageMapLevel4Entry
= PageMap
;
236 PageMapLevel5Entry
= NULL
;
237 if (m5LevelPagingNeeded
) {
239 // By architecture only one PageMapLevel5 exists - so lets allocate storage for it.
241 PageMapLevel5Entry
= PageMap
;
245 for ( IndexOfPml5Entries
= 0
246 ; IndexOfPml5Entries
< NumberOfPml5EntriesNeeded
247 ; IndexOfPml5Entries
++, PageMapLevel5Entry
++) {
249 // Each PML5 entry points to a page of PML4 entires.
250 // So lets allocate space for them and fill them in in the IndexOfPml4Entries loop.
251 // When 5-Level Paging is disabled, below allocation happens only once.
253 if (m5LevelPagingNeeded
) {
254 PageMapLevel4Entry
= (UINT64
*) ((*PageMapLevel5Entry
) & ~mAddressEncMask
& gPhyMask
);
255 if (PageMapLevel4Entry
== NULL
) {
256 PageMapLevel4Entry
= AllocatePageTableMemory (1);
257 ASSERT(PageMapLevel4Entry
!= NULL
);
258 ZeroMem (PageMapLevel4Entry
, EFI_PAGES_TO_SIZE(1));
260 *PageMapLevel5Entry
= (UINT64
)(UINTN
)PageMapLevel4Entry
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
264 for (IndexOfPml4Entries
= 0; IndexOfPml4Entries
< (NumberOfPml5EntriesNeeded
== 1 ? NumberOfPml4EntriesNeeded
: 512); IndexOfPml4Entries
++, PageMapLevel4Entry
++) {
266 // Each PML4 entry points to a page of Page Directory Pointer entries.
268 PageDirectoryPointerEntry
= (UINT64
*) ((*PageMapLevel4Entry
) & ~mAddressEncMask
& gPhyMask
);
269 if (PageDirectoryPointerEntry
== NULL
) {
270 PageDirectoryPointerEntry
= AllocatePageTableMemory (1);
271 ASSERT(PageDirectoryPointerEntry
!= NULL
);
272 ZeroMem (PageDirectoryPointerEntry
, EFI_PAGES_TO_SIZE(1));
274 *PageMapLevel4Entry
= (UINT64
)(UINTN
)PageDirectoryPointerEntry
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
277 if (m1GPageTableSupport
) {
278 PageDirectory1GEntry
= PageDirectoryPointerEntry
;
279 for (IndexOfPageDirectoryEntries
= 0; IndexOfPageDirectoryEntries
< 512; IndexOfPageDirectoryEntries
++, PageDirectory1GEntry
++, PageAddress
+= SIZE_1GB
) {
280 if (IndexOfPml4Entries
== 0 && IndexOfPageDirectoryEntries
< 4) {
282 // Skip the < 4G entries
287 // Fill in the Page Directory entries
289 *PageDirectory1GEntry
= PageAddress
| mAddressEncMask
| IA32_PG_PS
| PAGE_ATTRIBUTE_BITS
;
292 PageAddress
= BASE_4GB
;
293 for (IndexOfPdpEntries
= 0; IndexOfPdpEntries
< (NumberOfPml4EntriesNeeded
== 1 ? NumberOfPdpEntriesNeeded
: 512); IndexOfPdpEntries
++, PageDirectoryPointerEntry
++) {
294 if (IndexOfPml4Entries
== 0 && IndexOfPdpEntries
< 4) {
296 // Skip the < 4G entries
301 // Each Directory Pointer entries points to a page of Page Directory entires.
302 // So allocate space for them and fill them in in the IndexOfPageDirectoryEntries loop.
304 PageDirectoryEntry
= (UINT64
*) ((*PageDirectoryPointerEntry
) & ~mAddressEncMask
& gPhyMask
);
305 if (PageDirectoryEntry
== NULL
) {
306 PageDirectoryEntry
= AllocatePageTableMemory (1);
307 ASSERT(PageDirectoryEntry
!= NULL
);
308 ZeroMem (PageDirectoryEntry
, EFI_PAGES_TO_SIZE(1));
311 // Fill in a Page Directory Pointer Entries
313 *PageDirectoryPointerEntry
= (UINT64
)(UINTN
)PageDirectoryEntry
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
316 for (IndexOfPageDirectoryEntries
= 0; IndexOfPageDirectoryEntries
< 512; IndexOfPageDirectoryEntries
++, PageDirectoryEntry
++, PageAddress
+= SIZE_2MB
) {
318 // Fill in the Page Directory entries
320 *PageDirectoryEntry
= PageAddress
| mAddressEncMask
| IA32_PG_PS
| PAGE_ATTRIBUTE_BITS
;
329 Create PageTable for SMM use.
331 @return The address of PML4 (to set CR3).
339 EFI_PHYSICAL_ADDRESS Pages
;
341 LIST_ENTRY
*FreePage
;
343 UINTN PageFaultHandlerHookAddress
;
344 IA32_IDT_GATE_DESCRIPTOR
*IdtEntry
;
350 // Initialize spin lock
352 InitializeSpinLock (mPFLock
);
354 mCpuSmmRestrictedMemoryAccess
= PcdGetBool (PcdCpuSmmRestrictedMemoryAccess
);
355 m1GPageTableSupport
= Is1GPageSupport ();
356 m5LevelPagingNeeded
= Is5LevelPagingNeeded ();
357 mPhysicalAddressBits
= CalculateMaximumSupportAddress ();
358 PatchInstructionX86 (gPatch5LevelPagingNeeded
, m5LevelPagingNeeded
, 1);
359 DEBUG ((DEBUG_INFO
, "5LevelPaging Needed - %d\n", m5LevelPagingNeeded
));
360 DEBUG ((DEBUG_INFO
, "1GPageTable Support - %d\n", m1GPageTableSupport
));
361 DEBUG ((DEBUG_INFO
, "PcdCpuSmmRestrictedMemoryAccess - %d\n", mCpuSmmRestrictedMemoryAccess
));
362 DEBUG ((DEBUG_INFO
, "PhysicalAddressBits - %d\n", mPhysicalAddressBits
));
364 // Generate PAE page table for the first 4GB memory space
366 Pages
= Gen4GPageTable (FALSE
);
369 // Set IA32_PG_PMNT bit to mask this entry
371 PTEntry
= (UINT64
*)(UINTN
)Pages
;
372 for (Index
= 0; Index
< 4; Index
++) {
373 PTEntry
[Index
] |= IA32_PG_PMNT
;
377 // Fill Page-Table-Level4 (PML4) entry
379 Pml4Entry
= (UINT64
*)AllocatePageTableMemory (1);
380 ASSERT (Pml4Entry
!= NULL
);
381 *Pml4Entry
= Pages
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
382 ZeroMem (Pml4Entry
+ 1, EFI_PAGE_SIZE
- sizeof (*Pml4Entry
));
385 // Set sub-entries number
387 SetSubEntriesNum (Pml4Entry
, 3);
390 if (m5LevelPagingNeeded
) {
394 Pml5Entry
= (UINT64
*)AllocatePageTableMemory (1);
395 ASSERT (Pml5Entry
!= NULL
);
396 *Pml5Entry
= (UINTN
) Pml4Entry
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
397 ZeroMem (Pml5Entry
+ 1, EFI_PAGE_SIZE
- sizeof (*Pml5Entry
));
399 // Set sub-entries number
401 SetSubEntriesNum (Pml5Entry
, 1);
405 if (mCpuSmmRestrictedMemoryAccess
) {
407 // When access to non-SMRAM memory is restricted, create page table
408 // that covers all memory space.
410 SetStaticPageTable ((UINTN
)PTEntry
);
413 // Add pages to page pool
415 FreePage
= (LIST_ENTRY
*)AllocatePageTableMemory (PAGE_TABLE_PAGES
);
416 ASSERT (FreePage
!= NULL
);
417 for (Index
= 0; Index
< PAGE_TABLE_PAGES
; Index
++) {
418 InsertTailList (&mPagePool
, FreePage
);
419 FreePage
+= EFI_PAGE_SIZE
/ sizeof (*FreePage
);
423 if (FeaturePcdGet (PcdCpuSmmProfileEnable
) ||
424 HEAP_GUARD_NONSTOP_MODE
||
425 NULL_DETECTION_NONSTOP_MODE
) {
427 // Set own Page Fault entry instead of the default one, because SMM Profile
428 // feature depends on IRET instruction to do Single Step
430 PageFaultHandlerHookAddress
= (UINTN
)PageFaultIdtHandlerSmmProfile
;
431 IdtEntry
= (IA32_IDT_GATE_DESCRIPTOR
*) gcSmiIdtr
.Base
;
432 IdtEntry
+= EXCEPT_IA32_PAGE_FAULT
;
433 IdtEntry
->Bits
.OffsetLow
= (UINT16
)PageFaultHandlerHookAddress
;
434 IdtEntry
->Bits
.Reserved_0
= 0;
435 IdtEntry
->Bits
.GateType
= IA32_IDT_GATE_TYPE_INTERRUPT_32
;
436 IdtEntry
->Bits
.OffsetHigh
= (UINT16
)(PageFaultHandlerHookAddress
>> 16);
437 IdtEntry
->Bits
.OffsetUpper
= (UINT32
)(PageFaultHandlerHookAddress
>> 32);
438 IdtEntry
->Bits
.Reserved_1
= 0;
441 // Register Smm Page Fault Handler
443 Status
= SmmRegisterExceptionHandler (&mSmmCpuService
, EXCEPT_IA32_PAGE_FAULT
, SmiPFHandler
);
444 ASSERT_EFI_ERROR (Status
);
448 // Additional SMM IDT initialization for SMM stack guard
450 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
451 InitializeIDTSmmStackGuard ();
455 // Return the address of PML4/PML5 (to set CR3)
457 return (UINT32
)(UINTN
)PTEntry
;
461 Set access record in entry.
463 @param[in, out] Entry Pointer to entry
464 @param[in] Acc Access record value
469 IN OUT UINT64
*Entry
,
474 // Access record is saved in BIT9 to BIT11 (reserved field) in Entry
476 *Entry
= BitFieldWrite64 (*Entry
, 9, 11, Acc
);
480 Return access record in entry.
482 @param[in] Entry Pointer to entry
484 @return Access record value.
493 // Access record is saved in BIT9 to BIT11 (reserved field) in Entry
495 return BitFieldRead64 (*Entry
, 9, 11);
499 Return and update the access record in entry.
501 @param[in, out] Entry Pointer to entry
503 @return Access record value.
513 Acc
= GetAccNum (Entry
);
514 if ((*Entry
& IA32_PG_A
) != 0) {
516 // If this entry has been accessed, clear access flag in Entry and update access record
517 // to the initial value 7, adding ACC_MAX_BIT is to make it larger than others
519 *Entry
&= ~(UINT64
)(UINTN
)IA32_PG_A
;
520 SetAccNum (Entry
, 0x7);
521 return (0x7 + ACC_MAX_BIT
);
525 // If the access record is not the smallest value 0, minus 1 and update the access record field
527 SetAccNum (Entry
, Acc
- 1);
534 Reclaim free pages for PageFault handler.
536 Search the whole entries tree to find the leaf entry that has the smallest
537 access record value. Insert the page pointed by this leaf entry into the
538 page pool. And check its upper entries if need to be inserted into the page
562 UINT64 SubEntriesNum
;
565 UINT64
*ReleasePageAddress
;
567 BOOLEAN Enable5LevelPaging
;
569 UINT64 PFAddressPml5Index
;
570 UINT64 PFAddressPml4Index
;
571 UINT64 PFAddressPdptIndex
;
572 UINT64 PFAddressPdtIndex
;
583 ReleasePageAddress
= 0;
584 PFAddress
= AsmReadCr2 ();
585 PFAddressPml5Index
= BitFieldRead64 (PFAddress
, 48, 48 + 8);
586 PFAddressPml4Index
= BitFieldRead64 (PFAddress
, 39, 39 + 8);
587 PFAddressPdptIndex
= BitFieldRead64 (PFAddress
, 30, 30 + 8);
588 PFAddressPdtIndex
= BitFieldRead64 (PFAddress
, 21, 21 + 8);
590 Cr4
.UintN
= AsmReadCr4 ();
591 Enable5LevelPaging
= (BOOLEAN
) (Cr4
.Bits
.LA57
== 1);
592 Pml5
= (UINT64
*)(UINTN
)(AsmReadCr3 () & gPhyMask
);
594 if (!Enable5LevelPaging
) {
596 // Create one fake PML5 entry for 4-Level Paging
597 // so that the page table parsing logic only handles 5-Level page structure.
599 Pml5Entry
= (UINTN
) Pml5
| IA32_PG_P
;
604 // First, find the leaf entry has the smallest access record value
606 for (Pml5Index
= 0; Pml5Index
< (Enable5LevelPaging
? (EFI_PAGE_SIZE
/ sizeof (*Pml4
)) : 1); Pml5Index
++) {
607 if ((Pml5
[Pml5Index
] & IA32_PG_P
) == 0 || (Pml5
[Pml5Index
] & IA32_PG_PMNT
) != 0) {
609 // If the PML5 entry is not present or is masked, skip it
613 Pml4
= (UINT64
*)(UINTN
)(Pml5
[Pml5Index
] & gPhyMask
);
614 for (Pml4Index
= 0; Pml4Index
< EFI_PAGE_SIZE
/ sizeof (*Pml4
); Pml4Index
++) {
615 if ((Pml4
[Pml4Index
] & IA32_PG_P
) == 0 || (Pml4
[Pml4Index
] & IA32_PG_PMNT
) != 0) {
617 // If the PML4 entry is not present or is masked, skip it
621 Pdpt
= (UINT64
*)(UINTN
)(Pml4
[Pml4Index
] & ~mAddressEncMask
& gPhyMask
);
623 for (PdptIndex
= 0; PdptIndex
< EFI_PAGE_SIZE
/ sizeof (*Pdpt
); PdptIndex
++) {
624 if ((Pdpt
[PdptIndex
] & IA32_PG_P
) == 0 || (Pdpt
[PdptIndex
] & IA32_PG_PMNT
) != 0) {
626 // If the PDPT entry is not present or is masked, skip it
628 if ((Pdpt
[PdptIndex
] & IA32_PG_PMNT
) != 0) {
630 // If the PDPT entry is masked, we will ignore checking the PML4 entry
636 if ((Pdpt
[PdptIndex
] & IA32_PG_PS
) == 0) {
638 // It's not 1-GByte pages entry, it should be a PDPT entry,
639 // we will not check PML4 entry more
642 Pdt
= (UINT64
*)(UINTN
)(Pdpt
[PdptIndex
] & ~mAddressEncMask
& gPhyMask
);
644 for (PdtIndex
= 0; PdtIndex
< EFI_PAGE_SIZE
/ sizeof(*Pdt
); PdtIndex
++) {
645 if ((Pdt
[PdtIndex
] & IA32_PG_P
) == 0 || (Pdt
[PdtIndex
] & IA32_PG_PMNT
) != 0) {
647 // If the PD entry is not present or is masked, skip it
649 if ((Pdt
[PdtIndex
] & IA32_PG_PMNT
) != 0) {
651 // If the PD entry is masked, we will not PDPT entry more
657 if ((Pdt
[PdtIndex
] & IA32_PG_PS
) == 0) {
659 // It's not 2 MByte page table entry, it should be PD entry
660 // we will find the entry has the smallest access record value
663 if (PdtIndex
!= PFAddressPdtIndex
|| PdptIndex
!= PFAddressPdptIndex
||
664 Pml4Index
!= PFAddressPml4Index
|| Pml5Index
!= PFAddressPml5Index
) {
665 Acc
= GetAndUpdateAccNum (Pdt
+ PdtIndex
);
668 // If the PD entry has the smallest access record value,
669 // save the Page address to be released
676 ReleasePageAddress
= Pdt
+ PdtIndex
;
683 // If this PDPT entry has no PDT entries pointer to 4 KByte pages,
684 // it should only has the entries point to 2 MByte Pages
686 if (PdptIndex
!= PFAddressPdptIndex
|| Pml4Index
!= PFAddressPml4Index
||
687 Pml5Index
!= PFAddressPml5Index
) {
688 Acc
= GetAndUpdateAccNum (Pdpt
+ PdptIndex
);
691 // If the PDPT entry has the smallest access record value,
692 // save the Page address to be released
699 ReleasePageAddress
= Pdpt
+ PdptIndex
;
707 // If PML4 entry has no the PDPT entry pointer to 2 MByte pages,
708 // it should only has the entries point to 1 GByte Pages
710 if (Pml4Index
!= PFAddressPml4Index
|| Pml5Index
!= PFAddressPml5Index
) {
711 Acc
= GetAndUpdateAccNum (Pml4
+ Pml4Index
);
714 // If the PML4 entry has the smallest access record value,
715 // save the Page address to be released
722 ReleasePageAddress
= Pml4
+ Pml4Index
;
729 // Make sure one PML4/PDPT/PD entry is selected
731 ASSERT (MinAcc
!= (UINT64
)-1);
734 // Secondly, insert the page pointed by this entry into page pool and clear this entry
736 InsertTailList (&mPagePool
, (LIST_ENTRY
*)(UINTN
)(*ReleasePageAddress
& ~mAddressEncMask
& gPhyMask
));
737 *ReleasePageAddress
= 0;
740 // Lastly, check this entry's upper entries if need to be inserted into page pool
744 if (MinPdt
!= (UINTN
)-1) {
746 // If 4 KByte Page Table is released, check the PDPT entry
748 Pml4
= (UINT64
*) (UINTN
) (Pml5
[MinPml5
] & gPhyMask
);
749 Pdpt
= (UINT64
*)(UINTN
)(Pml4
[MinPml4
] & ~mAddressEncMask
& gPhyMask
);
750 SubEntriesNum
= GetSubEntriesNum(Pdpt
+ MinPdpt
);
751 if (SubEntriesNum
== 0 &&
752 (MinPdpt
!= PFAddressPdptIndex
|| MinPml4
!= PFAddressPml4Index
|| MinPml5
!= PFAddressPml5Index
)) {
754 // Release the empty Page Directory table if there was no more 4 KByte Page Table entry
755 // clear the Page directory entry
757 InsertTailList (&mPagePool
, (LIST_ENTRY
*)(UINTN
)(Pdpt
[MinPdpt
] & ~mAddressEncMask
& gPhyMask
));
760 // Go on checking the PML4 table
766 // Update the sub-entries filed in PDPT entry and exit
768 SetSubEntriesNum (Pdpt
+ MinPdpt
, (SubEntriesNum
- 1) & 0x1FF);
771 if (MinPdpt
!= (UINTN
)-1) {
773 // One 2MB Page Table is released or Page Directory table is released, check the PML4 entry
775 SubEntriesNum
= GetSubEntriesNum (Pml4
+ MinPml4
);
776 if (SubEntriesNum
== 0 && (MinPml4
!= PFAddressPml4Index
|| MinPml5
!= PFAddressPml5Index
)) {
778 // Release the empty PML4 table if there was no more 1G KByte Page Table entry
779 // clear the Page directory entry
781 InsertTailList (&mPagePool
, (LIST_ENTRY
*)(UINTN
)(Pml4
[MinPml4
] & ~mAddressEncMask
& gPhyMask
));
787 // Update the sub-entries filed in PML4 entry and exit
789 SetSubEntriesNum (Pml4
+ MinPml4
, (SubEntriesNum
- 1) & 0x1FF);
793 // PLM4 table has been released before, exit it
800 Allocate free Page for PageFault handler use.
802 @return Page address.
812 if (IsListEmpty (&mPagePool
)) {
814 // If page pool is empty, reclaim the used pages and insert one into page pool
820 // Get one free page and remove it from page pool
822 RetVal
= (UINT64
)(UINTN
)mPagePool
.ForwardLink
;
823 RemoveEntryList (mPagePool
.ForwardLink
);
825 // Clean this page and return
827 ZeroMem ((VOID
*)(UINTN
)RetVal
, EFI_PAGE_SIZE
);
832 Page Fault handler for SMM use.
836 SmiDefaultPFHandler (
841 UINT64
*PageTableTop
;
847 SMM_PAGE_SIZE_TYPE PageSize
;
852 BOOLEAN Enable5LevelPaging
;
856 // Set default SMM page attribute
858 PageSize
= SmmPageSize2M
;
863 PageTableTop
= (UINT64
*)(AsmReadCr3 () & gPhyMask
);
864 PFAddress
= AsmReadCr2 ();
866 Cr4
.UintN
= AsmReadCr4 ();
867 Enable5LevelPaging
= (BOOLEAN
) (Cr4
.Bits
.LA57
!= 0);
869 Status
= GetPlatformPageTableAttribute (PFAddress
, &PageSize
, &NumOfPages
, &PageAttribute
);
871 // If platform not support page table attribute, set default SMM page attribute
873 if (Status
!= EFI_SUCCESS
) {
874 PageSize
= SmmPageSize2M
;
878 if (PageSize
>= MaxSmmPageSizeType
) {
879 PageSize
= SmmPageSize2M
;
881 if (NumOfPages
> 512) {
888 // BIT12 to BIT20 is Page Table index
894 // BIT21 to BIT29 is Page Directory index
897 PageAttribute
|= (UINTN
)IA32_PG_PS
;
900 if (!m1GPageTableSupport
) {
901 DEBUG ((DEBUG_ERROR
, "1-GByte pages is not supported!"));
905 // BIT30 to BIT38 is Page Directory Pointer Table index
908 PageAttribute
|= (UINTN
)IA32_PG_PS
;
915 // If execute-disable is enabled, set NX bit
918 PageAttribute
|= IA32_PG_NX
;
921 for (Index
= 0; Index
< NumOfPages
; Index
++) {
922 PageTable
= PageTableTop
;
924 for (StartBit
= Enable5LevelPaging
? 48 : 39; StartBit
> EndBit
; StartBit
-= 9) {
925 PTIndex
= BitFieldRead64 (PFAddress
, StartBit
, StartBit
+ 8);
926 if ((PageTable
[PTIndex
] & IA32_PG_P
) == 0) {
928 // If the entry is not present, allocate one page from page pool for it
930 PageTable
[PTIndex
] = AllocPage () | mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
933 // Save the upper entry address
935 UpperEntry
= PageTable
+ PTIndex
;
938 // BIT9 to BIT11 of entry is used to save access record,
939 // initialize value is 7
941 PageTable
[PTIndex
] |= (UINT64
)IA32_PG_A
;
942 SetAccNum (PageTable
+ PTIndex
, 7);
943 PageTable
= (UINT64
*)(UINTN
)(PageTable
[PTIndex
] & ~mAddressEncMask
& gPhyMask
);
946 PTIndex
= BitFieldRead64 (PFAddress
, StartBit
, StartBit
+ 8);
947 if ((PageTable
[PTIndex
] & IA32_PG_P
) != 0) {
949 // Check if the entry has already existed, this issue may occur when the different
950 // size page entries created under the same entry
952 DEBUG ((DEBUG_ERROR
, "PageTable = %lx, PTIndex = %x, PageTable[PTIndex] = %lx\n", PageTable
, PTIndex
, PageTable
[PTIndex
]));
953 DEBUG ((DEBUG_ERROR
, "New page table overlapped with old page table!\n"));
957 // Fill the new entry
959 PageTable
[PTIndex
] = ((PFAddress
| mAddressEncMask
) & gPhyMask
& ~((1ull << EndBit
) - 1)) |
960 PageAttribute
| IA32_PG_A
| PAGE_ATTRIBUTE_BITS
;
961 if (UpperEntry
!= NULL
) {
962 SetSubEntriesNum (UpperEntry
, (GetSubEntriesNum (UpperEntry
) + 1) & 0x1FF);
965 // Get the next page address if we need to create more page tables
967 PFAddress
+= (1ull << EndBit
);
972 ThePage Fault handler wrapper for SMM use.
974 @param InterruptType Defines the type of interrupt or exception that
975 occurred on the processor.This parameter is processor architecture specific.
976 @param SystemContext A pointer to the processor context when
977 the interrupt occurred on the processor.
982 IN EFI_EXCEPTION_TYPE InterruptType
,
983 IN EFI_SYSTEM_CONTEXT SystemContext
987 UINTN GuardPageAddress
;
990 ASSERT (InterruptType
== EXCEPT_IA32_PAGE_FAULT
);
992 AcquireSpinLock (mPFLock
);
994 PFAddress
= AsmReadCr2 ();
996 if (mCpuSmmRestrictedMemoryAccess
&& (PFAddress
>= LShiftU64 (1, (mPhysicalAddressBits
- 1)))) {
997 DumpCpuContext (InterruptType
, SystemContext
);
998 DEBUG ((DEBUG_ERROR
, "Do not support address 0x%lx by processor!\n", PFAddress
));
1004 // If a page fault occurs in SMRAM range, it might be in a SMM stack guard page,
1005 // or SMM page protection violation.
1007 if ((PFAddress
>= mCpuHotPlugData
.SmrrBase
) &&
1008 (PFAddress
< (mCpuHotPlugData
.SmrrBase
+ mCpuHotPlugData
.SmrrSize
))) {
1009 DumpCpuContext (InterruptType
, SystemContext
);
1010 CpuIndex
= GetCpuIndex ();
1011 GuardPageAddress
= (mSmmStackArrayBase
+ EFI_PAGE_SIZE
+ CpuIndex
* mSmmStackSize
);
1012 if ((FeaturePcdGet (PcdCpuSmmStackGuard
)) &&
1013 (PFAddress
>= GuardPageAddress
) &&
1014 (PFAddress
< (GuardPageAddress
+ EFI_PAGE_SIZE
))) {
1015 DEBUG ((DEBUG_ERROR
, "SMM stack overflow!\n"));
1017 if ((SystemContext
.SystemContextX64
->ExceptionData
& IA32_PF_EC_ID
) != 0) {
1018 DEBUG ((DEBUG_ERROR
, "SMM exception at execution (0x%lx)\n", PFAddress
));
1020 DumpModuleInfoByIp (*(UINTN
*)(UINTN
)SystemContext
.SystemContextX64
->Rsp
);
1023 DEBUG ((DEBUG_ERROR
, "SMM exception at access (0x%lx)\n", PFAddress
));
1025 DumpModuleInfoByIp ((UINTN
)SystemContext
.SystemContextX64
->Rip
);
1029 if (HEAP_GUARD_NONSTOP_MODE
) {
1030 GuardPagePFHandler (SystemContext
.SystemContextX64
->ExceptionData
);
1039 // If a page fault occurs in non-SMRAM range.
1041 if ((PFAddress
< mCpuHotPlugData
.SmrrBase
) ||
1042 (PFAddress
>= mCpuHotPlugData
.SmrrBase
+ mCpuHotPlugData
.SmrrSize
)) {
1043 if ((SystemContext
.SystemContextX64
->ExceptionData
& IA32_PF_EC_ID
) != 0) {
1044 DumpCpuContext (InterruptType
, SystemContext
);
1045 DEBUG ((DEBUG_ERROR
, "Code executed on IP(0x%lx) out of SMM range after SMM is locked!\n", PFAddress
));
1047 DumpModuleInfoByIp (*(UINTN
*)(UINTN
)SystemContext
.SystemContextX64
->Rsp
);
1054 // If NULL pointer was just accessed
1056 if ((PcdGet8 (PcdNullPointerDetectionPropertyMask
) & BIT1
) != 0 &&
1057 (PFAddress
< EFI_PAGE_SIZE
)) {
1058 DumpCpuContext (InterruptType
, SystemContext
);
1059 DEBUG ((DEBUG_ERROR
, "!!! NULL pointer access !!!\n"));
1061 DumpModuleInfoByIp ((UINTN
)SystemContext
.SystemContextX64
->Rip
);
1064 if (NULL_DETECTION_NONSTOP_MODE
) {
1065 GuardPagePFHandler (SystemContext
.SystemContextX64
->ExceptionData
);
1073 if (mCpuSmmRestrictedMemoryAccess
&& IsSmmCommBufferForbiddenAddress (PFAddress
)) {
1074 DumpCpuContext (InterruptType
, SystemContext
);
1075 DEBUG ((DEBUG_ERROR
, "Access SMM communication forbidden address (0x%lx)!\n", PFAddress
));
1077 DumpModuleInfoByIp ((UINTN
)SystemContext
.SystemContextX64
->Rip
);
1084 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1085 SmmProfilePFHandler (
1086 SystemContext
.SystemContextX64
->Rip
,
1087 SystemContext
.SystemContextX64
->ExceptionData
1090 SmiDefaultPFHandler ();
1094 ReleaseSpinLock (mPFLock
);
1098 This function sets memory attribute for page table.
1101 SetPageTableAttributes (
1109 UINT64
*L1PageTable
;
1110 UINT64
*L2PageTable
;
1111 UINT64
*L3PageTable
;
1112 UINT64
*L4PageTable
;
1113 UINT64
*L5PageTable
;
1115 BOOLEAN PageTableSplitted
;
1118 BOOLEAN Enable5LevelPaging
;
1120 Cr4
.UintN
= AsmReadCr4 ();
1121 Enable5LevelPaging
= (BOOLEAN
) (Cr4
.Bits
.LA57
== 1);
1124 // Don't mark page table memory as read-only if
1125 // - no restriction on access to non-SMRAM memory; or
1126 // - SMM heap guard feature enabled; or
1127 // BIT2: SMM page guard enabled
1128 // BIT3: SMM pool guard enabled
1129 // - SMM profile feature enabled
1131 if (!mCpuSmmRestrictedMemoryAccess
||
1132 ((PcdGet8 (PcdHeapGuardPropertyMask
) & (BIT3
| BIT2
)) != 0) ||
1133 FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1135 // Restriction on access to non-SMRAM memory and heap guard could not be enabled at the same time.
1137 ASSERT (!(mCpuSmmRestrictedMemoryAccess
&&
1138 (PcdGet8 (PcdHeapGuardPropertyMask
) & (BIT3
| BIT2
)) != 0));
1141 // Restriction on access to non-SMRAM memory and SMM profile could not be enabled at the same time.
1143 ASSERT (!(mCpuSmmRestrictedMemoryAccess
&& FeaturePcdGet (PcdCpuSmmProfileEnable
)));
1147 DEBUG ((DEBUG_INFO
, "SetPageTableAttributes\n"));
1150 // Disable write protection, because we need mark page table to be write protected.
1151 // We need *write* page table memory, to mark itself to be *read only*.
1153 CetEnabled
= ((AsmReadCr4() & CR4_CET_ENABLE
) != 0) ? TRUE
: FALSE
;
1156 // CET must be disabled if WP is disabled.
1160 AsmWriteCr0 (AsmReadCr0() & ~CR0_WP
);
1163 DEBUG ((DEBUG_INFO
, "Start...\n"));
1164 PageTableSplitted
= FALSE
;
1166 if (Enable5LevelPaging
) {
1167 L5PageTable
= (UINT64
*)GetPageTableBase ();
1168 SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS
)(UINTN
)L5PageTable
, SIZE_4KB
, EFI_MEMORY_RO
, &IsSplitted
);
1169 PageTableSplitted
= (PageTableSplitted
|| IsSplitted
);
1172 for (Index5
= 0; Index5
< (Enable5LevelPaging
? SIZE_4KB
/sizeof(UINT64
) : 1); Index5
++) {
1173 if (Enable5LevelPaging
) {
1174 L4PageTable
= (UINT64
*)(UINTN
)(L5PageTable
[Index5
] & ~mAddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
1175 if (L4PageTable
== NULL
) {
1179 L4PageTable
= (UINT64
*)GetPageTableBase ();
1181 SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS
)(UINTN
)L4PageTable
, SIZE_4KB
, EFI_MEMORY_RO
, &IsSplitted
);
1182 PageTableSplitted
= (PageTableSplitted
|| IsSplitted
);
1184 for (Index4
= 0; Index4
< SIZE_4KB
/sizeof(UINT64
); Index4
++) {
1185 L3PageTable
= (UINT64
*)(UINTN
)(L4PageTable
[Index4
] & ~mAddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
1186 if (L3PageTable
== NULL
) {
1190 SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS
)(UINTN
)L3PageTable
, SIZE_4KB
, EFI_MEMORY_RO
, &IsSplitted
);
1191 PageTableSplitted
= (PageTableSplitted
|| IsSplitted
);
1193 for (Index3
= 0; Index3
< SIZE_4KB
/sizeof(UINT64
); Index3
++) {
1194 if ((L3PageTable
[Index3
] & IA32_PG_PS
) != 0) {
1198 L2PageTable
= (UINT64
*)(UINTN
)(L3PageTable
[Index3
] & ~mAddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
1199 if (L2PageTable
== NULL
) {
1203 SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS
)(UINTN
)L2PageTable
, SIZE_4KB
, EFI_MEMORY_RO
, &IsSplitted
);
1204 PageTableSplitted
= (PageTableSplitted
|| IsSplitted
);
1206 for (Index2
= 0; Index2
< SIZE_4KB
/sizeof(UINT64
); Index2
++) {
1207 if ((L2PageTable
[Index2
] & IA32_PG_PS
) != 0) {
1211 L1PageTable
= (UINT64
*)(UINTN
)(L2PageTable
[Index2
] & ~mAddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
1212 if (L1PageTable
== NULL
) {
1215 SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS
)(UINTN
)L1PageTable
, SIZE_4KB
, EFI_MEMORY_RO
, &IsSplitted
);
1216 PageTableSplitted
= (PageTableSplitted
|| IsSplitted
);
1221 } while (PageTableSplitted
);
1224 // Enable write protection, after page table updated.
1226 AsmWriteCr0 (AsmReadCr0() | CR0_WP
);
1238 This function reads CR2 register when on-demand paging is enabled.
1240 @param[out] *Cr2 Pointer to variable to hold CR2 register value.
1247 if (!mCpuSmmRestrictedMemoryAccess
) {
1249 // On-demand paging is enabled when access to non-SMRAM is not restricted.
1251 *Cr2
= AsmReadCr2 ();
1256 This function restores CR2 register when on-demand paging is enabled.
1258 @param[in] Cr2 Value to write into CR2 register.
1265 if (!mCpuSmmRestrictedMemoryAccess
) {
1267 // On-demand paging is enabled when access to non-SMRAM is not restricted.
1274 Return whether access to non-SMRAM is restricted.
1276 @retval TRUE Access to non-SMRAM is restricted.
1277 @retval FALSE Access to non-SMRAM is not restricted.
1280 IsRestrictedMemoryAccess (
1284 return mCpuSmmRestrictedMemoryAccess
;