2 Page table management support.
4 Copyright (c) 2017 - 2019, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
13 #include <Library/PeCoffGetEntryPointLib.h>
14 #include <Library/SerialPortLib.h>
15 #include <Library/SynchronizationLib.h>
16 #include <Library/PrintLib.h>
17 #include <Protocol/SmmBase2.h>
18 #include <Register/Intel/Cpuid.h>
19 #include <Register/Intel/Msr.h>
22 #include "CpuPageTable.h"
27 #define IA32_PG_P BIT0
28 #define IA32_PG_RW BIT1
29 #define IA32_PG_U BIT2
30 #define IA32_PG_WT BIT3
31 #define IA32_PG_CD BIT4
32 #define IA32_PG_A BIT5
33 #define IA32_PG_D BIT6
34 #define IA32_PG_PS BIT7
35 #define IA32_PG_PAT_2M BIT12
36 #define IA32_PG_PAT_4K IA32_PG_PS
37 #define IA32_PG_PMNT BIT62
38 #define IA32_PG_NX BIT63
40 #define PAGE_ATTRIBUTE_BITS (IA32_PG_D | IA32_PG_A | IA32_PG_U | IA32_PG_RW | IA32_PG_P)
42 // Bits 1, 2, 5, 6 are reserved in the IA32 PAE PDPTE
43 // X64 PAE PDPTE does not have such restriction
45 #define IA32_PAE_PDPTE_ATTRIBUTE_BITS (IA32_PG_P)
47 #define PAGE_PROGATE_BITS (IA32_PG_NX | PAGE_ATTRIBUTE_BITS)
49 #define PAGING_4K_MASK 0xFFF
50 #define PAGING_2M_MASK 0x1FFFFF
51 #define PAGING_1G_MASK 0x3FFFFFFF
53 #define PAGING_PAE_INDEX_MASK 0x1FF
55 #define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull
56 #define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull
57 #define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull
59 #define MAX_PF_ENTRY_COUNT 10
60 #define MAX_DEBUG_MESSAGE_LENGTH 0x100
61 #define IA32_PF_EC_ID BIT4
71 PAGE_ATTRIBUTE Attribute
;
74 } PAGE_ATTRIBUTE_TABLE
;
82 PAGE_ATTRIBUTE_TABLE mPageAttributeTable
[] = {
83 { Page4K
, SIZE_4KB
, PAGING_4K_ADDRESS_MASK_64
},
84 { Page2M
, SIZE_2MB
, PAGING_2M_ADDRESS_MASK_64
},
85 { Page1G
, SIZE_1GB
, PAGING_1G_ADDRESS_MASK_64
},
88 PAGE_TABLE_POOL
*mPageTablePool
= NULL
;
89 BOOLEAN mPageTablePoolLock
= FALSE
;
90 PAGE_TABLE_LIB_PAGING_CONTEXT mPagingContext
;
91 EFI_SMM_BASE2_PROTOCOL
*mSmmBase2
= NULL
;
94 // Record the page fault exception count for one instruction execution.
97 UINT64
*(*mLastPFEntryPointer
)[MAX_PF_ENTRY_COUNT
];
100 Check if current execution environment is in SMM mode or not, via
101 EFI_SMM_BASE2_PROTOCOL.
103 This is necessary because of the fact that MdePkg\Library\SmmMemoryAllocationLib
104 supports to free memory outside SMRAM. The library will call gBS->FreePool() or
105 gBS->FreePages() and then SetMemorySpaceAttributes interface in turn to change
106 memory paging attributes during free operation, if some memory related features
107 are enabled (like Heap Guard).
109 This means that SetMemorySpaceAttributes() has chance to run in SMM mode. This
110 will cause incorrect result because SMM mode always loads its own page tables,
111 which are usually different from DXE. This function can be used to detect such
112 situation and help to avoid further misoperations.
114 @retval TRUE In SMM mode.
115 @retval FALSE Not in SMM mode.
125 if (mSmmBase2
== NULL
) {
126 gBS
->LocateProtocol (&gEfiSmmBase2ProtocolGuid
, NULL
, (VOID
**)&mSmmBase2
);
129 if (mSmmBase2
!= NULL
) {
130 mSmmBase2
->InSmm (mSmmBase2
, &InSmm
);
134 // mSmmBase2->InSmm() can only detect if the caller is running in SMRAM
135 // or from SMM driver. It cannot tell if the caller is running in SMM mode.
136 // Check page table base address to guarantee that because SMM mode willl
137 // load its own page table.
140 mPagingContext
.ContextData
.X64
.PageTableBase
!= (UINT64
)AsmReadCr3 ());
144 Return current paging context.
146 @param[in,out] PagingContext The paging context.
149 GetCurrentPagingContext (
150 IN OUT PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
154 CPUID_EXTENDED_CPU_SIG_EDX RegEdx
;
155 MSR_IA32_EFER_REGISTER MsrEfer
;
159 UINTN
*PageTableBase
;
162 // Don't retrieve current paging context from processor if in SMM mode.
165 ZeroMem (&mPagingContext
, sizeof (mPagingContext
));
166 if (sizeof (UINTN
) == sizeof (UINT64
)) {
167 mPagingContext
.MachineType
= IMAGE_FILE_MACHINE_X64
;
169 mPagingContext
.MachineType
= IMAGE_FILE_MACHINE_I386
;
172 GetPagingDetails (&mPagingContext
.ContextData
, &PageTableBase
, &Attributes
);
174 Cr0
.UintN
= AsmReadCr0 ();
175 Cr4
.UintN
= AsmReadCr4 ();
177 if (Cr0
.Bits
.PG
!= 0) {
178 *PageTableBase
= (AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64
);
183 if (Cr0
.Bits
.WP
!= 0) {
184 *Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_WP_ENABLE
;
187 if (Cr4
.Bits
.PSE
!= 0) {
188 *Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PSE
;
191 if (Cr4
.Bits
.PAE
!= 0) {
192 *Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
;
195 if (Cr4
.Bits
.LA57
!= 0) {
196 *Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_5_LEVEL
;
199 AsmCpuid (CPUID_EXTENDED_FUNCTION
, &RegEax
, NULL
, NULL
, NULL
);
200 if (RegEax
>= CPUID_EXTENDED_CPU_SIG
) {
201 AsmCpuid (CPUID_EXTENDED_CPU_SIG
, NULL
, NULL
, NULL
, &RegEdx
.Uint32
);
203 if (RegEdx
.Bits
.NX
!= 0) {
205 MsrEfer
.Uint64
= AsmReadMsr64 (MSR_CORE_IA32_EFER
);
206 if (MsrEfer
.Bits
.NXE
!= 0) {
208 *Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED
;
212 if (RegEdx
.Bits
.Page1GB
!= 0) {
213 *Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAGE_1G_SUPPORT
;
219 // This can avoid getting SMM paging context if in SMM mode. We cannot assume
220 // SMM mode shares the same paging context as DXE.
222 CopyMem (PagingContext
, &mPagingContext
, sizeof (mPagingContext
));
226 Return length according to page attributes.
228 @param[in] PageAttributes The page attribute of the page entry.
230 @return The length of page entry.
233 PageAttributeToLength (
234 IN PAGE_ATTRIBUTE PageAttribute
239 for (Index
= 0; Index
< sizeof (mPageAttributeTable
)/sizeof (mPageAttributeTable
[0]); Index
++) {
240 if (PageAttribute
== mPageAttributeTable
[Index
].Attribute
) {
241 return (UINTN
)mPageAttributeTable
[Index
].Length
;
249 Return address mask according to page attributes.
251 @param[in] PageAttributes The page attribute of the page entry.
253 @return The address mask of page entry.
256 PageAttributeToMask (
257 IN PAGE_ATTRIBUTE PageAttribute
262 for (Index
= 0; Index
< sizeof (mPageAttributeTable
)/sizeof (mPageAttributeTable
[0]); Index
++) {
263 if (PageAttribute
== mPageAttributeTable
[Index
].Attribute
) {
264 return (UINTN
)mPageAttributeTable
[Index
].AddressMask
;
272 Return page table entry to match the address.
274 @param[in] PagingContext The paging context.
275 @param[in] Address The address to be checked.
276 @param[out] PageAttributes The page attribute of the page entry.
278 @return The page entry.
282 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
,
283 IN PHYSICAL_ADDRESS Address
,
284 OUT PAGE_ATTRIBUTE
*PageAttribute
297 UINT64 AddressEncMask
;
299 ASSERT (PagingContext
!= NULL
);
301 Index5
= ((UINTN
)RShiftU64 (Address
, 48)) & PAGING_PAE_INDEX_MASK
;
302 Index4
= ((UINTN
)RShiftU64 (Address
, 39)) & PAGING_PAE_INDEX_MASK
;
303 Index3
= ((UINTN
)Address
>> 30) & PAGING_PAE_INDEX_MASK
;
304 Index2
= ((UINTN
)Address
>> 21) & PAGING_PAE_INDEX_MASK
;
305 Index1
= ((UINTN
)Address
>> 12) & PAGING_PAE_INDEX_MASK
;
307 // Make sure AddressEncMask is contained to smallest supported address field.
309 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
310 if (AddressEncMask
== 0) {
311 AddressEncMask
= PcdGet64 (PcdTdxSharedBitMask
) & PAGING_1G_ADDRESS_MASK_64
;
314 if (PagingContext
->MachineType
== IMAGE_FILE_MACHINE_X64
) {
315 if ((PagingContext
->ContextData
.X64
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_5_LEVEL
) != 0) {
316 L5PageTable
= (UINT64
*)(UINTN
)PagingContext
->ContextData
.X64
.PageTableBase
;
317 if (L5PageTable
[Index5
] == 0) {
318 *PageAttribute
= PageNone
;
322 L4PageTable
= (UINT64
*)(UINTN
)(L5PageTable
[Index5
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
324 L4PageTable
= (UINT64
*)(UINTN
)PagingContext
->ContextData
.X64
.PageTableBase
;
327 if (L4PageTable
[Index4
] == 0) {
328 *PageAttribute
= PageNone
;
332 L3PageTable
= (UINT64
*)(UINTN
)(L4PageTable
[Index4
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
334 ASSERT ((PagingContext
->ContextData
.Ia32
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) != 0);
335 L3PageTable
= (UINT64
*)(UINTN
)PagingContext
->ContextData
.Ia32
.PageTableBase
;
338 if (L3PageTable
[Index3
] == 0) {
339 *PageAttribute
= PageNone
;
343 if ((L3PageTable
[Index3
] & IA32_PG_PS
) != 0) {
345 *PageAttribute
= Page1G
;
346 return &L3PageTable
[Index3
];
349 L2PageTable
= (UINT64
*)(UINTN
)(L3PageTable
[Index3
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
350 if (L2PageTable
[Index2
] == 0) {
351 *PageAttribute
= PageNone
;
355 if ((L2PageTable
[Index2
] & IA32_PG_PS
) != 0) {
357 *PageAttribute
= Page2M
;
358 return &L2PageTable
[Index2
];
362 L1PageTable
= (UINT64
*)(UINTN
)(L2PageTable
[Index2
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
363 if ((L1PageTable
[Index1
] == 0) && (Address
!= 0)) {
364 *PageAttribute
= PageNone
;
368 *PageAttribute
= Page4K
;
369 return &L1PageTable
[Index1
];
373 Return memory attributes of page entry.
375 @param[in] PageEntry The page entry.
377 @return Memory attributes of page entry.
380 GetAttributesFromPageEntry (
387 if ((*PageEntry
& IA32_PG_P
) == 0) {
388 Attributes
|= EFI_MEMORY_RP
;
391 if ((*PageEntry
& IA32_PG_RW
) == 0) {
392 Attributes
|= EFI_MEMORY_RO
;
395 if ((*PageEntry
& IA32_PG_NX
) != 0) {
396 Attributes
|= EFI_MEMORY_XP
;
403 Modify memory attributes of page entry.
405 @param[in] PagingContext The paging context.
406 @param[in] PageEntry The page entry.
407 @param[in] Attributes The bit mask of attributes to modify for the memory region.
408 @param[in] PageAction The page action.
409 @param[out] IsModified TRUE means page table modified. FALSE means page table not modified.
412 ConvertPageEntryAttribute (
413 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
,
414 IN UINT64
*PageEntry
,
415 IN UINT64 Attributes
,
416 IN PAGE_ACTION PageAction
,
417 OUT BOOLEAN
*IsModified
420 UINT64 CurrentPageEntry
;
422 UINT32
*PageAttributes
;
424 CurrentPageEntry
= *PageEntry
;
425 NewPageEntry
= CurrentPageEntry
;
426 if ((Attributes
& EFI_MEMORY_RP
) != 0) {
427 switch (PageAction
) {
428 case PageActionAssign
:
430 NewPageEntry
&= ~(UINT64
)IA32_PG_P
;
432 case PageActionClear
:
433 NewPageEntry
|= IA32_PG_P
;
437 switch (PageAction
) {
438 case PageActionAssign
:
439 NewPageEntry
|= IA32_PG_P
;
442 case PageActionClear
:
447 if ((Attributes
& EFI_MEMORY_RO
) != 0) {
448 switch (PageAction
) {
449 case PageActionAssign
:
451 NewPageEntry
&= ~(UINT64
)IA32_PG_RW
;
453 case PageActionClear
:
454 NewPageEntry
|= IA32_PG_RW
;
458 switch (PageAction
) {
459 case PageActionAssign
:
460 NewPageEntry
|= IA32_PG_RW
;
463 case PageActionClear
:
468 GetPagingDetails (&PagingContext
->ContextData
, NULL
, &PageAttributes
);
470 if ((*PageAttributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED
) != 0) {
471 if ((Attributes
& EFI_MEMORY_XP
) != 0) {
472 switch (PageAction
) {
473 case PageActionAssign
:
475 NewPageEntry
|= IA32_PG_NX
;
477 case PageActionClear
:
478 NewPageEntry
&= ~IA32_PG_NX
;
482 switch (PageAction
) {
483 case PageActionAssign
:
484 NewPageEntry
&= ~IA32_PG_NX
;
487 case PageActionClear
:
493 *PageEntry
= NewPageEntry
;
494 if (CurrentPageEntry
!= NewPageEntry
) {
496 DEBUG ((DEBUG_VERBOSE
, "ConvertPageEntryAttribute 0x%lx", CurrentPageEntry
));
497 DEBUG ((DEBUG_VERBOSE
, "->0x%lx\n", NewPageEntry
));
504 This function returns if there is need to split page entry.
506 @param[in] BaseAddress The base address to be checked.
507 @param[in] Length The length to be checked.
508 @param[in] PageEntry The page entry to be checked.
509 @param[in] PageAttribute The page attribute of the page entry.
511 @retval SplitAttributes on if there is need to split page entry.
515 IN PHYSICAL_ADDRESS BaseAddress
,
517 IN UINT64
*PageEntry
,
518 IN PAGE_ATTRIBUTE PageAttribute
521 UINT64 PageEntryLength
;
523 PageEntryLength
= PageAttributeToLength (PageAttribute
);
525 if (((BaseAddress
& (PageEntryLength
- 1)) == 0) && (Length
>= PageEntryLength
)) {
529 if (((BaseAddress
& PAGING_2M_MASK
) != 0) || (Length
< SIZE_2MB
)) {
537 This function splits one page entry to small page entries.
539 @param[in] PageEntry The page entry to be splitted.
540 @param[in] PageAttribute The page attribute of the page entry.
541 @param[in] SplitAttribute How to split the page entry.
542 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
544 @retval RETURN_SUCCESS The page entry is splitted.
545 @retval RETURN_UNSUPPORTED The page entry does not support to be splitted.
546 @retval RETURN_OUT_OF_RESOURCES No resource to split page entry.
550 IN UINT64
*PageEntry
,
551 IN PAGE_ATTRIBUTE PageAttribute
,
552 IN PAGE_ATTRIBUTE SplitAttribute
,
553 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc
557 UINT64
*NewPageEntry
;
559 UINT64 AddressEncMask
;
561 ASSERT (PageAttribute
== Page2M
|| PageAttribute
== Page1G
);
563 ASSERT (AllocatePagesFunc
!= NULL
);
565 // Make sure AddressEncMask is contained to smallest supported address field.
567 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
569 if (PageAttribute
== Page2M
) {
573 ASSERT (SplitAttribute
== Page4K
);
574 if (SplitAttribute
== Page4K
) {
575 NewPageEntry
= AllocatePagesFunc (1);
576 DEBUG ((DEBUG_VERBOSE
, "Split - 0x%x\n", NewPageEntry
));
577 if (NewPageEntry
== NULL
) {
578 return RETURN_OUT_OF_RESOURCES
;
581 BaseAddress
= *PageEntry
& ~AddressEncMask
& PAGING_2M_ADDRESS_MASK_64
;
582 for (Index
= 0; Index
< SIZE_4KB
/ sizeof (UINT64
); Index
++) {
583 NewPageEntry
[Index
] = (BaseAddress
+ SIZE_4KB
* Index
) | AddressEncMask
| ((*PageEntry
) & PAGE_PROGATE_BITS
);
586 (*PageEntry
) = (UINT64
)(UINTN
)NewPageEntry
| AddressEncMask
| ((*PageEntry
) & PAGE_ATTRIBUTE_BITS
);
587 return RETURN_SUCCESS
;
589 return RETURN_UNSUPPORTED
;
591 } else if (PageAttribute
== Page1G
) {
594 // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
596 ASSERT (SplitAttribute
== Page2M
|| SplitAttribute
== Page4K
);
597 if (((SplitAttribute
== Page2M
) || (SplitAttribute
== Page4K
))) {
598 NewPageEntry
= AllocatePagesFunc (1);
599 DEBUG ((DEBUG_VERBOSE
, "Split - 0x%x\n", NewPageEntry
));
600 if (NewPageEntry
== NULL
) {
601 return RETURN_OUT_OF_RESOURCES
;
604 BaseAddress
= *PageEntry
& ~AddressEncMask
& PAGING_1G_ADDRESS_MASK_64
;
605 for (Index
= 0; Index
< SIZE_4KB
/ sizeof (UINT64
); Index
++) {
606 NewPageEntry
[Index
] = (BaseAddress
+ SIZE_2MB
* Index
) | AddressEncMask
| IA32_PG_PS
| ((*PageEntry
) & PAGE_PROGATE_BITS
);
609 (*PageEntry
) = (UINT64
)(UINTN
)NewPageEntry
| AddressEncMask
| ((*PageEntry
) & PAGE_ATTRIBUTE_BITS
);
610 return RETURN_SUCCESS
;
612 return RETURN_UNSUPPORTED
;
615 return RETURN_UNSUPPORTED
;
620 Check the WP status in CR0 register. This bit is used to lock or unlock write
621 access to pages marked as read-only.
623 @retval TRUE Write protection is enabled.
624 @retval FALSE Write protection is disabled.
627 IsReadOnlyPageWriteProtected (
634 // To avoid unforseen consequences, don't touch paging settings in SMM mode
638 Cr0
.UintN
= AsmReadCr0 ();
639 return (BOOLEAN
)(Cr0
.Bits
.WP
!= 0);
646 Disable Write Protect on pages marked as read-only.
649 DisableReadOnlyPageWriteProtect (
656 // To avoid unforseen consequences, don't touch paging settings in SMM mode
660 Cr0
.UintN
= AsmReadCr0 ();
662 AsmWriteCr0 (Cr0
.UintN
);
667 Enable Write Protect on pages marked as read-only.
670 EnableReadOnlyPageWriteProtect (
677 // To avoid unforseen consequences, don't touch paging settings in SMM mode
681 Cr0
.UintN
= AsmReadCr0 ();
683 AsmWriteCr0 (Cr0
.UintN
);
688 This function modifies the page attributes for the memory region specified by BaseAddress and
689 Length from their current attributes to the attributes specified by Attributes.
691 Caller should make sure BaseAddress and Length is at page boundary.
693 @param[in] PagingContext The paging context. NULL means get page table from current CPU context.
694 @param[in] BaseAddress The physical address that is the start address of a memory region.
695 @param[in] Length The size in bytes of the memory region.
696 @param[in] Attributes The bit mask of attributes to modify for the memory region.
697 @param[in] PageAction The page action.
698 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
699 NULL mean page split is unsupported.
700 @param[out] IsSplitted TRUE means page table splitted. FALSE means page table not splitted.
701 @param[out] IsModified TRUE means page table modified. FALSE means page table not modified.
703 @retval RETURN_SUCCESS The attributes were modified for the memory region.
704 @retval RETURN_ACCESS_DENIED The attributes for the memory resource range specified by
705 BaseAddress and Length cannot be modified.
706 @retval RETURN_INVALID_PARAMETER Length is zero.
707 Attributes specified an illegal combination of attributes that
708 cannot be set together.
709 @retval RETURN_OUT_OF_RESOURCES There are not enough system resources to modify the attributes of
710 the memory resource range.
711 @retval RETURN_UNSUPPORTED The processor does not support one or more bytes of the memory
712 resource range specified by BaseAddress and Length.
713 The bit mask of attributes is not support for the memory resource
714 range specified by BaseAddress and Length.
717 ConvertMemoryPageAttributes (
718 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext OPTIONAL
,
719 IN PHYSICAL_ADDRESS BaseAddress
,
721 IN UINT64 Attributes
,
722 IN PAGE_ACTION PageAction
,
723 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc OPTIONAL
,
724 OUT BOOLEAN
*IsSplitted OPTIONAL
,
725 OUT BOOLEAN
*IsModified OPTIONAL
728 PAGE_TABLE_LIB_PAGING_CONTEXT CurrentPagingContext
;
730 PAGE_ATTRIBUTE PageAttribute
;
731 UINTN PageEntryLength
;
732 PAGE_ATTRIBUTE SplitAttribute
;
733 RETURN_STATUS Status
;
734 BOOLEAN IsEntryModified
;
737 if ((BaseAddress
& (SIZE_4KB
- 1)) != 0) {
738 DEBUG ((DEBUG_ERROR
, "BaseAddress(0x%lx) is not aligned!\n", BaseAddress
));
739 return EFI_UNSUPPORTED
;
742 if ((Length
& (SIZE_4KB
- 1)) != 0) {
743 DEBUG ((DEBUG_ERROR
, "Length(0x%lx) is not aligned!\n", Length
));
744 return EFI_UNSUPPORTED
;
748 DEBUG ((DEBUG_ERROR
, "Length is 0!\n"));
749 return RETURN_INVALID_PARAMETER
;
752 if ((Attributes
& ~EFI_MEMORY_ATTRIBUTE_MASK
) != 0) {
753 DEBUG ((DEBUG_ERROR
, "Attributes(0x%lx) has unsupported bit\n", Attributes
));
754 return EFI_UNSUPPORTED
;
757 if (PagingContext
== NULL
) {
758 GetCurrentPagingContext (&CurrentPagingContext
);
760 CopyMem (&CurrentPagingContext
, PagingContext
, sizeof (CurrentPagingContext
));
763 switch (CurrentPagingContext
.MachineType
) {
764 case IMAGE_FILE_MACHINE_I386
:
765 if (CurrentPagingContext
.ContextData
.Ia32
.PageTableBase
== 0) {
766 if (Attributes
== 0) {
769 DEBUG ((DEBUG_ERROR
, "PageTable is 0!\n"));
770 return EFI_UNSUPPORTED
;
774 if ((CurrentPagingContext
.ContextData
.Ia32
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) == 0) {
775 DEBUG ((DEBUG_ERROR
, "Non-PAE Paging!\n"));
776 return EFI_UNSUPPORTED
;
779 if ((BaseAddress
+ Length
) > BASE_4GB
) {
780 DEBUG ((DEBUG_ERROR
, "Beyond 4GB memory in 32-bit mode!\n"));
781 return EFI_UNSUPPORTED
;
785 case IMAGE_FILE_MACHINE_X64
:
786 ASSERT (CurrentPagingContext
.ContextData
.X64
.PageTableBase
!= 0);
790 return EFI_UNSUPPORTED
;
794 // DEBUG ((DEBUG_ERROR, "ConvertMemoryPageAttributes(%x) - %016lx, %016lx, %02lx\n", IsSet, BaseAddress, Length, Attributes));
796 if (IsSplitted
!= NULL
) {
800 if (IsModified
!= NULL
) {
804 if (AllocatePagesFunc
== NULL
) {
805 AllocatePagesFunc
= AllocatePageTableMemory
;
809 // Make sure that the page table is changeable.
811 IsWpEnabled
= IsReadOnlyPageWriteProtected ();
813 DisableReadOnlyPageWriteProtect ();
817 // Below logic is to check 2M/4K page to make sure we do not waste memory.
819 Status
= EFI_SUCCESS
;
820 while (Length
!= 0) {
821 PageEntry
= GetPageTableEntry (&CurrentPagingContext
, BaseAddress
, &PageAttribute
);
822 if (PageEntry
== NULL
) {
823 Status
= RETURN_UNSUPPORTED
;
827 PageEntryLength
= PageAttributeToLength (PageAttribute
);
828 SplitAttribute
= NeedSplitPage (BaseAddress
, Length
, PageEntry
, PageAttribute
);
829 if (SplitAttribute
== PageNone
) {
830 ConvertPageEntryAttribute (&CurrentPagingContext
, PageEntry
, Attributes
, PageAction
, &IsEntryModified
);
831 if (IsEntryModified
) {
832 if (IsModified
!= NULL
) {
838 // Convert success, move to next
840 BaseAddress
+= PageEntryLength
;
841 Length
-= PageEntryLength
;
843 if (AllocatePagesFunc
== NULL
) {
844 Status
= RETURN_UNSUPPORTED
;
848 Status
= SplitPage (PageEntry
, PageAttribute
, SplitAttribute
, AllocatePagesFunc
);
849 if (RETURN_ERROR (Status
)) {
850 Status
= RETURN_UNSUPPORTED
;
854 if (IsSplitted
!= NULL
) {
858 if (IsModified
!= NULL
) {
863 // Just split current page
864 // Convert success in next around
871 // Restore page table write protection, if any.
874 EnableReadOnlyPageWriteProtect ();
881 This function assigns the page attributes for the memory region specified by BaseAddress and
882 Length from their current attributes to the attributes specified by Attributes.
884 Caller should make sure BaseAddress and Length is at page boundary.
886 Caller need guarantee the TPL <= TPL_NOTIFY, if there is split page request.
888 @param[in] PagingContext The paging context. NULL means get page table from current CPU context.
889 @param[in] BaseAddress The physical address that is the start address of a memory region.
890 @param[in] Length The size in bytes of the memory region.
891 @param[in] Attributes The bit mask of attributes to set for the memory region.
892 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
893 NULL mean page split is unsupported.
895 @retval RETURN_SUCCESS The attributes were cleared for the memory region.
896 @retval RETURN_ACCESS_DENIED The attributes for the memory resource range specified by
897 BaseAddress and Length cannot be modified.
898 @retval RETURN_INVALID_PARAMETER Length is zero.
899 Attributes specified an illegal combination of attributes that
900 cannot be set together.
901 @retval RETURN_OUT_OF_RESOURCES There are not enough system resources to modify the attributes of
902 the memory resource range.
903 @retval RETURN_UNSUPPORTED The processor does not support one or more bytes of the memory
904 resource range specified by BaseAddress and Length.
905 The bit mask of attributes is not support for the memory resource
906 range specified by BaseAddress and Length.
910 AssignMemoryPageAttributes (
911 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext OPTIONAL
,
912 IN PHYSICAL_ADDRESS BaseAddress
,
914 IN UINT64 Attributes
,
915 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc OPTIONAL
918 RETURN_STATUS Status
;
922 // DEBUG((DEBUG_INFO, "AssignMemoryPageAttributes: 0x%lx - 0x%lx (0x%lx)\n", BaseAddress, Length, Attributes));
923 Status
= ConvertMemoryPageAttributes (PagingContext
, BaseAddress
, Length
, Attributes
, PageActionAssign
, AllocatePagesFunc
, &IsSplitted
, &IsModified
);
924 if (!EFI_ERROR (Status
)) {
925 if ((PagingContext
== NULL
) && IsModified
) {
927 // Flush TLB as last step.
929 // Note: Since APs will always init CR3 register in HLT loop mode or do
930 // TLB flush in MWAIT loop mode, there's no need to flush TLB for them
941 Check if Execute Disable feature is enabled or not.
944 IsExecuteDisableEnabled (
948 MSR_CORE_IA32_EFER_REGISTER MsrEfer
;
950 MsrEfer
.Uint64
= AsmReadMsr64 (MSR_IA32_EFER
);
951 return (MsrEfer
.Bits
.NXE
== 1);
955 Update GCD memory space attributes according to current page table setup.
958 RefreshGcdMemoryAttributesFromPaging (
963 UINTN NumberOfDescriptors
;
964 EFI_GCD_MEMORY_SPACE_DESCRIPTOR
*MemorySpaceMap
;
965 PAGE_TABLE_LIB_PAGING_CONTEXT PagingContext
;
966 PAGE_ATTRIBUTE PageAttribute
;
969 UINT64 MemorySpaceLength
;
972 UINT64 PageStartAddress
;
975 UINT64 NewAttributes
;
979 // Assuming that memory space map returned is sorted already; otherwise sort
980 // them in the order of lowest address to highest address.
982 Status
= gDS
->GetMemorySpaceMap (&NumberOfDescriptors
, &MemorySpaceMap
);
983 ASSERT_EFI_ERROR (Status
);
985 GetCurrentPagingContext (&PagingContext
);
992 if (IsExecuteDisableEnabled ()) {
993 Capabilities
= EFI_MEMORY_RO
| EFI_MEMORY_RP
| EFI_MEMORY_XP
;
995 Capabilities
= EFI_MEMORY_RO
| EFI_MEMORY_RP
;
998 for (Index
= 0; Index
< NumberOfDescriptors
; Index
++) {
999 if (MemorySpaceMap
[Index
].GcdMemoryType
== EfiGcdMemoryTypeNonExistent
) {
1004 // Sync the actual paging related capabilities back to GCD service first.
1005 // As a side effect (good one), this can also help to avoid unnecessary
1006 // memory map entries due to the different capabilities of the same type
1007 // memory, such as multiple RT_CODE and RT_DATA entries in memory map,
1008 // which could cause boot failure of some old Linux distro (before v4.3).
1010 Status
= gDS
->SetMemorySpaceCapabilities (
1011 MemorySpaceMap
[Index
].BaseAddress
,
1012 MemorySpaceMap
[Index
].Length
,
1013 MemorySpaceMap
[Index
].Capabilities
| Capabilities
1015 if (EFI_ERROR (Status
)) {
1017 // If we cannot update the capabilities, we cannot update its
1018 // attributes either. So just simply skip current block of memory.
1022 "Failed to update capability: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
1024 MemorySpaceMap
[Index
].BaseAddress
,
1025 MemorySpaceMap
[Index
].BaseAddress
+ MemorySpaceMap
[Index
].Length
- 1,
1026 MemorySpaceMap
[Index
].Capabilities
,
1027 MemorySpaceMap
[Index
].Capabilities
| Capabilities
1032 if (MemorySpaceMap
[Index
].BaseAddress
>= (BaseAddress
+ PageLength
)) {
1034 // Current memory space starts at a new page. Resetting PageLength will
1035 // trigger a retrieval of page attributes at new address.
1040 // In case current memory space is not adjacent to last one
1042 PageLength
-= (MemorySpaceMap
[Index
].BaseAddress
- BaseAddress
);
1046 // Sync actual page attributes to GCD
1048 BaseAddress
= MemorySpaceMap
[Index
].BaseAddress
;
1049 MemorySpaceLength
= MemorySpaceMap
[Index
].Length
;
1050 while (MemorySpaceLength
> 0) {
1051 if (PageLength
== 0) {
1052 PageEntry
= GetPageTableEntry (&PagingContext
, BaseAddress
, &PageAttribute
);
1053 if (PageEntry
== NULL
) {
1058 // Note current memory space might start in the middle of a page
1060 PageStartAddress
= (*PageEntry
) & (UINT64
)PageAttributeToMask (PageAttribute
);
1061 PageLength
= PageAttributeToLength (PageAttribute
) - (BaseAddress
- PageStartAddress
);
1062 Attributes
= GetAttributesFromPageEntry (PageEntry
);
1065 Length
= MIN (PageLength
, MemorySpaceLength
);
1066 if (Attributes
!= (MemorySpaceMap
[Index
].Attributes
&
1067 EFI_MEMORY_ATTRIBUTE_MASK
))
1069 NewAttributes
= (MemorySpaceMap
[Index
].Attributes
&
1070 ~EFI_MEMORY_ATTRIBUTE_MASK
) | Attributes
;
1071 Status
= gDS
->SetMemorySpaceAttributes (
1076 ASSERT_EFI_ERROR (Status
);
1079 "Updated memory space attribute: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
1082 BaseAddress
+ Length
- 1,
1083 MemorySpaceMap
[Index
].Attributes
,
1088 PageLength
-= Length
;
1089 MemorySpaceLength
-= Length
;
1090 BaseAddress
+= Length
;
1094 FreePool (MemorySpaceMap
);
1098 Initialize a buffer pool for page table use only.
1100 To reduce the potential split operation on page table, the pages reserved for
1101 page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
1102 at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
1103 initialized with number of pages greater than or equal to the given PoolPages.
1105 Once the pages in the pool are used up, this method should be called again to
1106 reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. Usually this won't happen
1109 @param[in] PoolPages The least page number of the pool to be created.
1111 @retval TRUE The pool is initialized successfully.
1112 @retval FALSE The memory is out of resource.
1115 InitializePageTablePool (
1123 // Do not allow re-entrance.
1125 if (mPageTablePoolLock
) {
1129 mPageTablePoolLock
= TRUE
;
1133 // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
1136 PoolPages
+= 1; // Add one page for header.
1137 PoolPages
= ((PoolPages
- 1) / PAGE_TABLE_POOL_UNIT_PAGES
+ 1) *
1138 PAGE_TABLE_POOL_UNIT_PAGES
;
1139 Buffer
= AllocateAlignedPages (PoolPages
, PAGE_TABLE_POOL_ALIGNMENT
);
1140 if (Buffer
== NULL
) {
1141 DEBUG ((DEBUG_ERROR
, "ERROR: Out of aligned pages\r\n"));
1147 "Paging: added %lu pages to page table pool\r\n",
1152 // Link all pools into a list for easier track later.
1154 if (mPageTablePool
== NULL
) {
1155 mPageTablePool
= Buffer
;
1156 mPageTablePool
->NextPool
= mPageTablePool
;
1158 ((PAGE_TABLE_POOL
*)Buffer
)->NextPool
= mPageTablePool
->NextPool
;
1159 mPageTablePool
->NextPool
= Buffer
;
1160 mPageTablePool
= Buffer
;
1164 // Reserve one page for pool header.
1166 mPageTablePool
->FreePages
= PoolPages
- 1;
1167 mPageTablePool
->Offset
= EFI_PAGES_TO_SIZE (1);
1170 // Mark the whole pool pages as read-only.
1172 ConvertMemoryPageAttributes (
1174 (PHYSICAL_ADDRESS
)(UINTN
)Buffer
,
1175 EFI_PAGES_TO_SIZE (PoolPages
),
1178 AllocatePageTableMemory
,
1182 ASSERT (IsModified
== TRUE
);
1185 mPageTablePoolLock
= FALSE
;
1190 This API provides a way to allocate memory for page table.
1192 This API can be called more than once to allocate memory for page tables.
1194 Allocates the number of 4KB pages and returns a pointer to the allocated
1195 buffer. The buffer returned is aligned on a 4KB boundary.
1197 If Pages is 0, then NULL is returned.
1198 If there is not enough memory remaining to satisfy the request, then NULL is
1201 @param Pages The number of 4 KB pages to allocate.
1203 @return A pointer to the allocated buffer or NULL if allocation fails.
1208 AllocatePageTableMemory (
1219 // Renew the pool if necessary.
1221 if ((mPageTablePool
== NULL
) ||
1222 (Pages
> mPageTablePool
->FreePages
))
1224 if (!InitializePageTablePool (Pages
)) {
1229 Buffer
= (UINT8
*)mPageTablePool
+ mPageTablePool
->Offset
;
1231 mPageTablePool
->Offset
+= EFI_PAGES_TO_SIZE (Pages
);
1232 mPageTablePool
->FreePages
-= Pages
;
1238 Special handler for #DB exception, which will restore the page attributes
1239 (not-present). It should work with #PF handler which will set pages to
1242 @param ExceptionType Exception type.
1243 @param SystemContext Pointer to EFI_SYSTEM_CONTEXT.
1248 DebugExceptionHandler (
1249 IN EFI_EXCEPTION_TYPE ExceptionType
,
1250 IN EFI_SYSTEM_CONTEXT SystemContext
1255 BOOLEAN IsWpEnabled
;
1257 MpInitLibWhoAmI (&CpuIndex
);
1260 // Clear last PF entries
1262 IsWpEnabled
= IsReadOnlyPageWriteProtected ();
1264 DisableReadOnlyPageWriteProtect ();
1267 for (PFEntry
= 0; PFEntry
< mPFEntryCount
[CpuIndex
]; PFEntry
++) {
1268 if (mLastPFEntryPointer
[CpuIndex
][PFEntry
] != NULL
) {
1269 *mLastPFEntryPointer
[CpuIndex
][PFEntry
] &= ~(UINT64
)IA32_PG_P
;
1274 EnableReadOnlyPageWriteProtect ();
1278 // Reset page fault exception count for next page fault.
1280 mPFEntryCount
[CpuIndex
] = 0;
1288 // Clear TF in EFLAGS
1290 if (mPagingContext
.MachineType
== IMAGE_FILE_MACHINE_I386
) {
1291 SystemContext
.SystemContextIa32
->Eflags
&= (UINT32
) ~BIT8
;
1293 SystemContext
.SystemContextX64
->Rflags
&= (UINT64
) ~BIT8
;
1298 Special handler for #PF exception, which will set the pages which caused
1299 #PF to be 'present'. The attribute of those pages should be restored in
1300 the subsequent #DB handler.
1302 @param ExceptionType Exception type.
1303 @param SystemContext Pointer to EFI_SYSTEM_CONTEXT.
1308 PageFaultExceptionHandler (
1309 IN EFI_EXCEPTION_TYPE ExceptionType
,
1310 IN EFI_SYSTEM_CONTEXT SystemContext
1315 PAGE_TABLE_LIB_PAGING_CONTEXT PagingContext
;
1316 PAGE_ATTRIBUTE PageAttribute
;
1322 BOOLEAN NonStopMode
;
1324 PFAddress
= AsmReadCr2 () & ~EFI_PAGE_MASK
;
1325 if (PFAddress
< BASE_4KB
) {
1326 NonStopMode
= NULL_DETECTION_NONSTOP_MODE
? TRUE
: FALSE
;
1328 NonStopMode
= HEAP_GUARD_NONSTOP_MODE
? TRUE
: FALSE
;
1332 MpInitLibWhoAmI (&CpuIndex
);
1333 GetCurrentPagingContext (&PagingContext
);
1335 // Memory operation cross page boundary, like "rep mov" instruction, will
1336 // cause infinite loop between this and Debug Trap handler. We have to make
1337 // sure that current page and the page followed are both in PRESENT state.
1340 while (PageNumber
> 0) {
1341 PageEntry
= GetPageTableEntry (&PagingContext
, PFAddress
, &PageAttribute
);
1342 ASSERT (PageEntry
!= NULL
);
1344 if (PageEntry
!= NULL
) {
1345 Attributes
= GetAttributesFromPageEntry (PageEntry
);
1346 if ((Attributes
& EFI_MEMORY_RP
) != 0) {
1347 Attributes
&= ~EFI_MEMORY_RP
;
1348 Status
= AssignMemoryPageAttributes (
1355 if (!EFI_ERROR (Status
)) {
1356 Index
= mPFEntryCount
[CpuIndex
];
1358 // Re-retrieve page entry because above calling might update page
1359 // table due to table split.
1361 PageEntry
= GetPageTableEntry (&PagingContext
, PFAddress
, &PageAttribute
);
1362 mLastPFEntryPointer
[CpuIndex
][Index
++] = PageEntry
;
1363 mPFEntryCount
[CpuIndex
] = Index
;
1368 PFAddress
+= EFI_PAGE_SIZE
;
1374 // Initialize the serial port before dumping.
1376 SerialPortInitialize ();
1378 // Display ExceptionType, CPU information and Image information
1380 DumpCpuContext (ExceptionType
, SystemContext
);
1385 if (mPagingContext
.MachineType
== IMAGE_FILE_MACHINE_I386
) {
1386 SystemContext
.SystemContextIa32
->Eflags
|= (UINT32
)BIT8
;
1388 SystemContext
.SystemContextX64
->Rflags
|= (UINT64
)BIT8
;
1396 Initialize the Page Table lib.
1399 InitializePageTableLib (
1403 PAGE_TABLE_LIB_PAGING_CONTEXT CurrentPagingContext
;
1405 UINTN
*PageTableBase
;
1407 GetCurrentPagingContext (&CurrentPagingContext
);
1409 GetPagingDetails (&CurrentPagingContext
.ContextData
, &PageTableBase
, &Attributes
);
1412 // Reserve memory of page tables for future uses, if paging is enabled.
1414 if ((*PageTableBase
!= 0) &&
1415 ((*Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) != 0))
1417 DisableReadOnlyPageWriteProtect ();
1418 InitializePageTablePool (1);
1419 EnableReadOnlyPageWriteProtect ();
1422 if (HEAP_GUARD_NONSTOP_MODE
|| NULL_DETECTION_NONSTOP_MODE
) {
1423 mPFEntryCount
= (UINTN
*)AllocateZeroPool (sizeof (UINTN
) * mNumberOfProcessors
);
1424 ASSERT (mPFEntryCount
!= NULL
);
1426 mLastPFEntryPointer
= (UINT64
*(*)[MAX_PF_ENTRY_COUNT
])
1427 AllocateZeroPool (sizeof (mLastPFEntryPointer
[0]) * mNumberOfProcessors
);
1428 ASSERT (mLastPFEntryPointer
!= NULL
);
1431 DEBUG ((DEBUG_INFO
, "CurrentPagingContext:\n"));
1432 DEBUG ((DEBUG_INFO
, " MachineType - 0x%x\n", CurrentPagingContext
.MachineType
));
1433 DEBUG ((DEBUG_INFO
, " PageTableBase - 0x%Lx\n", (UINT64
)*PageTableBase
));
1434 DEBUG ((DEBUG_INFO
, " Attributes - 0x%x\n", *Attributes
));