2 Page table management support.
4 Copyright (c) 2017 - 2019, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
13 #include <Library/PeCoffGetEntryPointLib.h>
14 #include <Library/SerialPortLib.h>
15 #include <Library/SynchronizationLib.h>
16 #include <Library/PrintLib.h>
17 #include <Protocol/SmmBase2.h>
18 #include <Register/Intel/Cpuid.h>
19 #include <Register/Intel/Msr.h>
22 #include "CpuPageTable.h"
27 #define IA32_PG_P BIT0
28 #define IA32_PG_RW BIT1
29 #define IA32_PG_U BIT2
30 #define IA32_PG_WT BIT3
31 #define IA32_PG_CD BIT4
32 #define IA32_PG_A BIT5
33 #define IA32_PG_D BIT6
34 #define IA32_PG_PS BIT7
35 #define IA32_PG_PAT_2M BIT12
36 #define IA32_PG_PAT_4K IA32_PG_PS
37 #define IA32_PG_PMNT BIT62
38 #define IA32_PG_NX BIT63
40 #define PAGE_ATTRIBUTE_BITS (IA32_PG_D | IA32_PG_A | IA32_PG_U | IA32_PG_RW | IA32_PG_P)
41 #define PAGE_ATTRIBUTE_BITS_POST_SPLIT (IA32_PG_RW | IA32_PG_P)
44 // Bits 1, 2, 5, 6 are reserved in the IA32 PAE PDPTE
45 // X64 PAE PDPTE does not have such restriction
47 #define IA32_PAE_PDPTE_ATTRIBUTE_BITS (IA32_PG_P)
49 #define PAGE_PROGATE_BITS (IA32_PG_NX | PAGE_ATTRIBUTE_BITS)
51 #define PAGING_4K_MASK 0xFFF
52 #define PAGING_2M_MASK 0x1FFFFF
53 #define PAGING_1G_MASK 0x3FFFFFFF
55 #define PAGING_PAE_INDEX_MASK 0x1FF
57 #define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull
58 #define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull
59 #define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull
61 #define MAX_PF_ENTRY_COUNT 10
62 #define MAX_DEBUG_MESSAGE_LENGTH 0x100
63 #define IA32_PF_EC_ID BIT4
73 PAGE_ATTRIBUTE Attribute
;
76 } PAGE_ATTRIBUTE_TABLE
;
84 PAGE_ATTRIBUTE_TABLE mPageAttributeTable
[] = {
85 { Page4K
, SIZE_4KB
, PAGING_4K_ADDRESS_MASK_64
},
86 { Page2M
, SIZE_2MB
, PAGING_2M_ADDRESS_MASK_64
},
87 { Page1G
, SIZE_1GB
, PAGING_1G_ADDRESS_MASK_64
},
90 PAGE_TABLE_POOL
*mPageTablePool
= NULL
;
91 BOOLEAN mPageTablePoolLock
= FALSE
;
92 PAGE_TABLE_LIB_PAGING_CONTEXT mPagingContext
;
93 EFI_SMM_BASE2_PROTOCOL
*mSmmBase2
= NULL
;
96 // Record the page fault exception count for one instruction execution.
99 UINT64
*(*mLastPFEntryPointer
)[MAX_PF_ENTRY_COUNT
];
102 Check if current execution environment is in SMM mode or not, via
103 EFI_SMM_BASE2_PROTOCOL.
105 This is necessary because of the fact that MdePkg\Library\SmmMemoryAllocationLib
106 supports to free memory outside SMRAM. The library will call gBS->FreePool() or
107 gBS->FreePages() and then SetMemorySpaceAttributes interface in turn to change
108 memory paging attributes during free operation, if some memory related features
109 are enabled (like Heap Guard).
111 This means that SetMemorySpaceAttributes() has chance to run in SMM mode. This
112 will cause incorrect result because SMM mode always loads its own page tables,
113 which are usually different from DXE. This function can be used to detect such
114 situation and help to avoid further misoperations.
116 @retval TRUE In SMM mode.
117 @retval FALSE Not in SMM mode.
127 if (mSmmBase2
== NULL
) {
128 gBS
->LocateProtocol (&gEfiSmmBase2ProtocolGuid
, NULL
, (VOID
**)&mSmmBase2
);
131 if (mSmmBase2
!= NULL
) {
132 mSmmBase2
->InSmm (mSmmBase2
, &InSmm
);
136 // mSmmBase2->InSmm() can only detect if the caller is running in SMRAM
137 // or from SMM driver. It cannot tell if the caller is running in SMM mode.
138 // Check page table base address to guarantee that because SMM mode willl
139 // load its own page table.
142 mPagingContext
.ContextData
.X64
.PageTableBase
!= (UINT64
)AsmReadCr3 ());
146 Return current paging context.
148 @param[in,out] PagingContext The paging context.
151 GetCurrentPagingContext (
152 IN OUT PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
156 CPUID_EXTENDED_CPU_SIG_EDX RegEdx
;
157 MSR_IA32_EFER_REGISTER MsrEfer
;
161 UINTN
*PageTableBase
;
164 // Don't retrieve current paging context from processor if in SMM mode.
167 ZeroMem (&mPagingContext
, sizeof (mPagingContext
));
168 if (sizeof (UINTN
) == sizeof (UINT64
)) {
169 mPagingContext
.MachineType
= IMAGE_FILE_MACHINE_X64
;
171 mPagingContext
.MachineType
= IMAGE_FILE_MACHINE_I386
;
174 GetPagingDetails (&mPagingContext
.ContextData
, &PageTableBase
, &Attributes
);
176 Cr0
.UintN
= AsmReadCr0 ();
177 Cr4
.UintN
= AsmReadCr4 ();
179 if (Cr0
.Bits
.PG
!= 0) {
180 *PageTableBase
= (AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64
);
185 if (Cr0
.Bits
.WP
!= 0) {
186 *Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_WP_ENABLE
;
189 if (Cr4
.Bits
.PSE
!= 0) {
190 *Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PSE
;
193 if (Cr4
.Bits
.PAE
!= 0) {
194 *Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
;
197 if (Cr4
.Bits
.LA57
!= 0) {
198 *Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_5_LEVEL
;
201 AsmCpuid (CPUID_EXTENDED_FUNCTION
, &RegEax
, NULL
, NULL
, NULL
);
202 if (RegEax
>= CPUID_EXTENDED_CPU_SIG
) {
203 AsmCpuid (CPUID_EXTENDED_CPU_SIG
, NULL
, NULL
, NULL
, &RegEdx
.Uint32
);
205 if (RegEdx
.Bits
.NX
!= 0) {
207 MsrEfer
.Uint64
= AsmReadMsr64 (MSR_CORE_IA32_EFER
);
208 if (MsrEfer
.Bits
.NXE
!= 0) {
210 *Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED
;
214 if (RegEdx
.Bits
.Page1GB
!= 0) {
215 *Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAGE_1G_SUPPORT
;
221 // This can avoid getting SMM paging context if in SMM mode. We cannot assume
222 // SMM mode shares the same paging context as DXE.
224 CopyMem (PagingContext
, &mPagingContext
, sizeof (mPagingContext
));
228 Return length according to page attributes.
230 @param[in] PageAttributes The page attribute of the page entry.
232 @return The length of page entry.
235 PageAttributeToLength (
236 IN PAGE_ATTRIBUTE PageAttribute
241 for (Index
= 0; Index
< sizeof (mPageAttributeTable
)/sizeof (mPageAttributeTable
[0]); Index
++) {
242 if (PageAttribute
== mPageAttributeTable
[Index
].Attribute
) {
243 return (UINTN
)mPageAttributeTable
[Index
].Length
;
251 Return address mask according to page attributes.
253 @param[in] PageAttributes The page attribute of the page entry.
255 @return The address mask of page entry.
258 PageAttributeToMask (
259 IN PAGE_ATTRIBUTE PageAttribute
264 for (Index
= 0; Index
< sizeof (mPageAttributeTable
)/sizeof (mPageAttributeTable
[0]); Index
++) {
265 if (PageAttribute
== mPageAttributeTable
[Index
].Attribute
) {
266 return (UINTN
)mPageAttributeTable
[Index
].AddressMask
;
274 Return page table entry to match the address.
276 @param[in] PagingContext The paging context.
277 @param[in] Address The address to be checked.
278 @param[out] PageAttributes The page attribute of the page entry.
280 @return The page entry.
284 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
,
285 IN PHYSICAL_ADDRESS Address
,
286 OUT PAGE_ATTRIBUTE
*PageAttribute
299 UINT64 AddressEncMask
;
301 ASSERT (PagingContext
!= NULL
);
303 Index5
= ((UINTN
)RShiftU64 (Address
, 48)) & PAGING_PAE_INDEX_MASK
;
304 Index4
= ((UINTN
)RShiftU64 (Address
, 39)) & PAGING_PAE_INDEX_MASK
;
305 Index3
= ((UINTN
)Address
>> 30) & PAGING_PAE_INDEX_MASK
;
306 Index2
= ((UINTN
)Address
>> 21) & PAGING_PAE_INDEX_MASK
;
307 Index1
= ((UINTN
)Address
>> 12) & PAGING_PAE_INDEX_MASK
;
309 // Make sure AddressEncMask is contained to smallest supported address field.
311 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
312 if (AddressEncMask
== 0) {
313 AddressEncMask
= PcdGet64 (PcdTdxSharedBitMask
) & PAGING_1G_ADDRESS_MASK_64
;
316 if (PagingContext
->MachineType
== IMAGE_FILE_MACHINE_X64
) {
317 if ((PagingContext
->ContextData
.X64
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_5_LEVEL
) != 0) {
318 L5PageTable
= (UINT64
*)(UINTN
)PagingContext
->ContextData
.X64
.PageTableBase
;
319 if (L5PageTable
[Index5
] == 0) {
320 *PageAttribute
= PageNone
;
324 L4PageTable
= (UINT64
*)(UINTN
)(L5PageTable
[Index5
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
326 L4PageTable
= (UINT64
*)(UINTN
)PagingContext
->ContextData
.X64
.PageTableBase
;
329 if (L4PageTable
[Index4
] == 0) {
330 *PageAttribute
= PageNone
;
334 L3PageTable
= (UINT64
*)(UINTN
)(L4PageTable
[Index4
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
336 ASSERT ((PagingContext
->ContextData
.Ia32
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) != 0);
337 L3PageTable
= (UINT64
*)(UINTN
)PagingContext
->ContextData
.Ia32
.PageTableBase
;
340 if (L3PageTable
[Index3
] == 0) {
341 *PageAttribute
= PageNone
;
345 if ((L3PageTable
[Index3
] & IA32_PG_PS
) != 0) {
347 *PageAttribute
= Page1G
;
348 return &L3PageTable
[Index3
];
351 L2PageTable
= (UINT64
*)(UINTN
)(L3PageTable
[Index3
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
352 if (L2PageTable
[Index2
] == 0) {
353 *PageAttribute
= PageNone
;
357 if ((L2PageTable
[Index2
] & IA32_PG_PS
) != 0) {
359 *PageAttribute
= Page2M
;
360 return &L2PageTable
[Index2
];
364 L1PageTable
= (UINT64
*)(UINTN
)(L2PageTable
[Index2
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
365 if ((L1PageTable
[Index1
] == 0) && (Address
!= 0)) {
366 *PageAttribute
= PageNone
;
370 *PageAttribute
= Page4K
;
371 return &L1PageTable
[Index1
];
375 Return memory attributes of page entry.
377 @param[in] PageEntry The page entry.
379 @return Memory attributes of page entry.
382 GetAttributesFromPageEntry (
389 if ((*PageEntry
& IA32_PG_P
) == 0) {
390 Attributes
|= EFI_MEMORY_RP
;
393 if ((*PageEntry
& IA32_PG_RW
) == 0) {
394 Attributes
|= EFI_MEMORY_RO
;
397 if ((*PageEntry
& IA32_PG_NX
) != 0) {
398 Attributes
|= EFI_MEMORY_XP
;
405 Modify memory attributes of page entry.
407 @param[in] PagingContext The paging context.
408 @param[in] PageEntry The page entry.
409 @param[in] Attributes The bit mask of attributes to modify for the memory region.
410 @param[in] PageAction The page action.
411 @param[out] IsModified TRUE means page table modified. FALSE means page table not modified.
414 ConvertPageEntryAttribute (
415 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
,
416 IN UINT64
*PageEntry
,
417 IN UINT64 Attributes
,
418 IN PAGE_ACTION PageAction
,
419 OUT BOOLEAN
*IsModified
422 UINT64 CurrentPageEntry
;
424 UINT32
*PageAttributes
;
426 CurrentPageEntry
= *PageEntry
;
427 NewPageEntry
= CurrentPageEntry
;
428 if ((Attributes
& EFI_MEMORY_RP
) != 0) {
429 switch (PageAction
) {
430 case PageActionAssign
:
432 NewPageEntry
&= ~(UINT64
)IA32_PG_P
;
434 case PageActionClear
:
435 NewPageEntry
|= IA32_PG_P
;
439 switch (PageAction
) {
440 case PageActionAssign
:
441 NewPageEntry
|= IA32_PG_P
;
444 case PageActionClear
:
449 if ((Attributes
& EFI_MEMORY_RO
) != 0) {
450 switch (PageAction
) {
451 case PageActionAssign
:
453 NewPageEntry
&= ~(UINT64
)IA32_PG_RW
;
455 case PageActionClear
:
456 NewPageEntry
|= IA32_PG_RW
;
460 switch (PageAction
) {
461 case PageActionAssign
:
462 NewPageEntry
|= IA32_PG_RW
;
465 case PageActionClear
:
470 GetPagingDetails (&PagingContext
->ContextData
, NULL
, &PageAttributes
);
472 if ((*PageAttributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED
) != 0) {
473 if ((Attributes
& EFI_MEMORY_XP
) != 0) {
474 switch (PageAction
) {
475 case PageActionAssign
:
477 NewPageEntry
|= IA32_PG_NX
;
479 case PageActionClear
:
480 NewPageEntry
&= ~IA32_PG_NX
;
484 switch (PageAction
) {
485 case PageActionAssign
:
486 NewPageEntry
&= ~IA32_PG_NX
;
489 case PageActionClear
:
495 *PageEntry
= NewPageEntry
;
496 if (CurrentPageEntry
!= NewPageEntry
) {
498 DEBUG ((DEBUG_VERBOSE
, "ConvertPageEntryAttribute 0x%lx", CurrentPageEntry
));
499 DEBUG ((DEBUG_VERBOSE
, "->0x%lx\n", NewPageEntry
));
506 This function returns if there is need to split page entry.
508 @param[in] BaseAddress The base address to be checked.
509 @param[in] Length The length to be checked.
510 @param[in] PageEntry The page entry to be checked.
511 @param[in] PageAttribute The page attribute of the page entry.
513 @retval SplitAttributes on if there is need to split page entry.
517 IN PHYSICAL_ADDRESS BaseAddress
,
519 IN UINT64
*PageEntry
,
520 IN PAGE_ATTRIBUTE PageAttribute
523 UINT64 PageEntryLength
;
525 PageEntryLength
= PageAttributeToLength (PageAttribute
);
527 if (((BaseAddress
& (PageEntryLength
- 1)) == 0) && (Length
>= PageEntryLength
)) {
531 if (((BaseAddress
& PAGING_2M_MASK
) != 0) || (Length
< SIZE_2MB
)) {
539 This function splits one page entry to small page entries.
541 @param[in] PageEntry The page entry to be splitted.
542 @param[in] PageAttribute The page attribute of the page entry.
543 @param[in] SplitAttribute How to split the page entry.
544 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
546 @retval RETURN_SUCCESS The page entry is splitted.
547 @retval RETURN_UNSUPPORTED The page entry does not support to be splitted.
548 @retval RETURN_OUT_OF_RESOURCES No resource to split page entry.
552 IN UINT64
*PageEntry
,
553 IN PAGE_ATTRIBUTE PageAttribute
,
554 IN PAGE_ATTRIBUTE SplitAttribute
,
555 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc
559 UINT64
*NewPageEntry
;
561 UINT64 AddressEncMask
;
563 ASSERT (PageAttribute
== Page2M
|| PageAttribute
== Page1G
);
565 ASSERT (AllocatePagesFunc
!= NULL
);
567 // Make sure AddressEncMask is contained to smallest supported address field.
569 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
571 if (PageAttribute
== Page2M
) {
575 ASSERT (SplitAttribute
== Page4K
);
576 if (SplitAttribute
== Page4K
) {
577 NewPageEntry
= AllocatePagesFunc (1);
578 DEBUG ((DEBUG_VERBOSE
, "Split - 0x%x\n", NewPageEntry
));
579 if (NewPageEntry
== NULL
) {
580 return RETURN_OUT_OF_RESOURCES
;
583 BaseAddress
= *PageEntry
& ~AddressEncMask
& PAGING_2M_ADDRESS_MASK_64
;
584 for (Index
= 0; Index
< SIZE_4KB
/ sizeof (UINT64
); Index
++) {
585 NewPageEntry
[Index
] = (BaseAddress
+ SIZE_4KB
* Index
) | AddressEncMask
| ((*PageEntry
) & PAGE_PROGATE_BITS
);
588 (*PageEntry
) = (UINT64
)(UINTN
)NewPageEntry
| AddressEncMask
| PAGE_ATTRIBUTE_BITS_POST_SPLIT
;
589 return RETURN_SUCCESS
;
591 return RETURN_UNSUPPORTED
;
593 } else if (PageAttribute
== Page1G
) {
596 // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
598 ASSERT (SplitAttribute
== Page2M
|| SplitAttribute
== Page4K
);
599 if (((SplitAttribute
== Page2M
) || (SplitAttribute
== Page4K
))) {
600 NewPageEntry
= AllocatePagesFunc (1);
601 DEBUG ((DEBUG_VERBOSE
, "Split - 0x%x\n", NewPageEntry
));
602 if (NewPageEntry
== NULL
) {
603 return RETURN_OUT_OF_RESOURCES
;
606 BaseAddress
= *PageEntry
& ~AddressEncMask
& PAGING_1G_ADDRESS_MASK_64
;
607 for (Index
= 0; Index
< SIZE_4KB
/ sizeof (UINT64
); Index
++) {
608 NewPageEntry
[Index
] = (BaseAddress
+ SIZE_2MB
* Index
) | AddressEncMask
| IA32_PG_PS
| ((*PageEntry
) & PAGE_PROGATE_BITS
);
611 (*PageEntry
) = (UINT64
)(UINTN
)NewPageEntry
| AddressEncMask
| PAGE_ATTRIBUTE_BITS_POST_SPLIT
;
612 return RETURN_SUCCESS
;
614 return RETURN_UNSUPPORTED
;
617 return RETURN_UNSUPPORTED
;
622 Check the WP status in CR0 register. This bit is used to lock or unlock write
623 access to pages marked as read-only.
625 @retval TRUE Write protection is enabled.
626 @retval FALSE Write protection is disabled.
629 IsReadOnlyPageWriteProtected (
636 // To avoid unforseen consequences, don't touch paging settings in SMM mode
640 Cr0
.UintN
= AsmReadCr0 ();
641 return (BOOLEAN
)(Cr0
.Bits
.WP
!= 0);
648 Disable Write Protect on pages marked as read-only.
651 DisableReadOnlyPageWriteProtect (
658 // To avoid unforseen consequences, don't touch paging settings in SMM mode
662 Cr0
.UintN
= AsmReadCr0 ();
664 AsmWriteCr0 (Cr0
.UintN
);
669 Enable Write Protect on pages marked as read-only.
672 EnableReadOnlyPageWriteProtect (
679 // To avoid unforseen consequences, don't touch paging settings in SMM mode
683 Cr0
.UintN
= AsmReadCr0 ();
685 AsmWriteCr0 (Cr0
.UintN
);
690 This function modifies the page attributes for the memory region specified by BaseAddress and
691 Length from their current attributes to the attributes specified by Attributes.
693 Caller should make sure BaseAddress and Length is at page boundary.
695 @param[in] PagingContext The paging context. NULL means get page table from current CPU context.
696 @param[in] BaseAddress The physical address that is the start address of a memory region.
697 @param[in] Length The size in bytes of the memory region.
698 @param[in] Attributes The bit mask of attributes to modify for the memory region.
699 @param[in] PageAction The page action.
700 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
701 NULL mean page split is unsupported.
702 @param[out] IsSplitted TRUE means page table splitted. FALSE means page table not splitted.
703 @param[out] IsModified TRUE means page table modified. FALSE means page table not modified.
705 @retval RETURN_SUCCESS The attributes were modified for the memory region.
706 @retval RETURN_ACCESS_DENIED The attributes for the memory resource range specified by
707 BaseAddress and Length cannot be modified.
708 @retval RETURN_INVALID_PARAMETER Length is zero.
709 Attributes specified an illegal combination of attributes that
710 cannot be set together.
711 @retval RETURN_OUT_OF_RESOURCES There are not enough system resources to modify the attributes of
712 the memory resource range.
713 @retval RETURN_UNSUPPORTED The processor does not support one or more bytes of the memory
714 resource range specified by BaseAddress and Length.
715 The bit mask of attributes is not support for the memory resource
716 range specified by BaseAddress and Length.
719 ConvertMemoryPageAttributes (
720 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext OPTIONAL
,
721 IN PHYSICAL_ADDRESS BaseAddress
,
723 IN UINT64 Attributes
,
724 IN PAGE_ACTION PageAction
,
725 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc OPTIONAL
,
726 OUT BOOLEAN
*IsSplitted OPTIONAL
,
727 OUT BOOLEAN
*IsModified OPTIONAL
730 PAGE_TABLE_LIB_PAGING_CONTEXT CurrentPagingContext
;
732 PAGE_ATTRIBUTE PageAttribute
;
733 UINTN PageEntryLength
;
734 PAGE_ATTRIBUTE SplitAttribute
;
735 RETURN_STATUS Status
;
736 BOOLEAN IsEntryModified
;
739 if ((BaseAddress
& (SIZE_4KB
- 1)) != 0) {
740 DEBUG ((DEBUG_ERROR
, "BaseAddress(0x%lx) is not aligned!\n", BaseAddress
));
741 return EFI_UNSUPPORTED
;
744 if ((Length
& (SIZE_4KB
- 1)) != 0) {
745 DEBUG ((DEBUG_ERROR
, "Length(0x%lx) is not aligned!\n", Length
));
746 return EFI_UNSUPPORTED
;
750 DEBUG ((DEBUG_ERROR
, "Length is 0!\n"));
751 return RETURN_INVALID_PARAMETER
;
754 if ((Attributes
& ~EFI_MEMORY_ATTRIBUTE_MASK
) != 0) {
755 DEBUG ((DEBUG_ERROR
, "Attributes(0x%lx) has unsupported bit\n", Attributes
));
756 return EFI_UNSUPPORTED
;
759 if (PagingContext
== NULL
) {
760 GetCurrentPagingContext (&CurrentPagingContext
);
762 CopyMem (&CurrentPagingContext
, PagingContext
, sizeof (CurrentPagingContext
));
765 switch (CurrentPagingContext
.MachineType
) {
766 case IMAGE_FILE_MACHINE_I386
:
767 if (CurrentPagingContext
.ContextData
.Ia32
.PageTableBase
== 0) {
768 if (Attributes
== 0) {
771 DEBUG ((DEBUG_ERROR
, "PageTable is 0!\n"));
772 return EFI_UNSUPPORTED
;
776 if ((CurrentPagingContext
.ContextData
.Ia32
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) == 0) {
777 DEBUG ((DEBUG_ERROR
, "Non-PAE Paging!\n"));
778 return EFI_UNSUPPORTED
;
781 if ((BaseAddress
+ Length
) > BASE_4GB
) {
782 DEBUG ((DEBUG_ERROR
, "Beyond 4GB memory in 32-bit mode!\n"));
783 return EFI_UNSUPPORTED
;
787 case IMAGE_FILE_MACHINE_X64
:
788 ASSERT (CurrentPagingContext
.ContextData
.X64
.PageTableBase
!= 0);
792 return EFI_UNSUPPORTED
;
796 // DEBUG ((DEBUG_ERROR, "ConvertMemoryPageAttributes(%x) - %016lx, %016lx, %02lx\n", IsSet, BaseAddress, Length, Attributes));
798 if (IsSplitted
!= NULL
) {
802 if (IsModified
!= NULL
) {
806 if (AllocatePagesFunc
== NULL
) {
807 AllocatePagesFunc
= AllocatePageTableMemory
;
811 // Make sure that the page table is changeable.
813 IsWpEnabled
= IsReadOnlyPageWriteProtected ();
815 DisableReadOnlyPageWriteProtect ();
819 // Below logic is to check 2M/4K page to make sure we do not waste memory.
821 Status
= EFI_SUCCESS
;
822 while (Length
!= 0) {
823 PageEntry
= GetPageTableEntry (&CurrentPagingContext
, BaseAddress
, &PageAttribute
);
824 if (PageEntry
== NULL
) {
825 Status
= RETURN_UNSUPPORTED
;
829 PageEntryLength
= PageAttributeToLength (PageAttribute
);
830 SplitAttribute
= NeedSplitPage (BaseAddress
, Length
, PageEntry
, PageAttribute
);
831 if (SplitAttribute
== PageNone
) {
832 ConvertPageEntryAttribute (&CurrentPagingContext
, PageEntry
, Attributes
, PageAction
, &IsEntryModified
);
833 if (IsEntryModified
) {
834 if (IsModified
!= NULL
) {
840 // Convert success, move to next
842 BaseAddress
+= PageEntryLength
;
843 Length
-= PageEntryLength
;
845 if (AllocatePagesFunc
== NULL
) {
846 Status
= RETURN_UNSUPPORTED
;
850 Status
= SplitPage (PageEntry
, PageAttribute
, SplitAttribute
, AllocatePagesFunc
);
851 if (RETURN_ERROR (Status
)) {
852 Status
= RETURN_UNSUPPORTED
;
856 if (IsSplitted
!= NULL
) {
860 if (IsModified
!= NULL
) {
865 // Just split current page
866 // Convert success in next around
873 // Restore page table write protection, if any.
876 EnableReadOnlyPageWriteProtect ();
883 This function assigns the page attributes for the memory region specified by BaseAddress and
884 Length from their current attributes to the attributes specified by Attributes.
886 Caller should make sure BaseAddress and Length is at page boundary.
888 Caller need guarantee the TPL <= TPL_NOTIFY, if there is split page request.
890 @param[in] PagingContext The paging context. NULL means get page table from current CPU context.
891 @param[in] BaseAddress The physical address that is the start address of a memory region.
892 @param[in] Length The size in bytes of the memory region.
893 @param[in] Attributes The bit mask of attributes to set for the memory region.
894 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
895 NULL mean page split is unsupported.
897 @retval RETURN_SUCCESS The attributes were cleared for the memory region.
898 @retval RETURN_ACCESS_DENIED The attributes for the memory resource range specified by
899 BaseAddress and Length cannot be modified.
900 @retval RETURN_INVALID_PARAMETER Length is zero.
901 Attributes specified an illegal combination of attributes that
902 cannot be set together.
903 @retval RETURN_OUT_OF_RESOURCES There are not enough system resources to modify the attributes of
904 the memory resource range.
905 @retval RETURN_UNSUPPORTED The processor does not support one or more bytes of the memory
906 resource range specified by BaseAddress and Length.
907 The bit mask of attributes is not support for the memory resource
908 range specified by BaseAddress and Length.
912 AssignMemoryPageAttributes (
913 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext OPTIONAL
,
914 IN PHYSICAL_ADDRESS BaseAddress
,
916 IN UINT64 Attributes
,
917 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc OPTIONAL
920 RETURN_STATUS Status
;
924 // DEBUG((DEBUG_INFO, "AssignMemoryPageAttributes: 0x%lx - 0x%lx (0x%lx)\n", BaseAddress, Length, Attributes));
925 Status
= ConvertMemoryPageAttributes (PagingContext
, BaseAddress
, Length
, Attributes
, PageActionAssign
, AllocatePagesFunc
, &IsSplitted
, &IsModified
);
926 if (!EFI_ERROR (Status
)) {
927 if ((PagingContext
== NULL
) && IsModified
) {
929 // Flush TLB as last step.
931 // Note: Since APs will always init CR3 register in HLT loop mode or do
932 // TLB flush in MWAIT loop mode, there's no need to flush TLB for them
943 Check if Execute Disable feature is enabled or not.
946 IsExecuteDisableEnabled (
950 MSR_CORE_IA32_EFER_REGISTER MsrEfer
;
952 MsrEfer
.Uint64
= AsmReadMsr64 (MSR_IA32_EFER
);
953 return (MsrEfer
.Bits
.NXE
== 1);
957 Update GCD memory space attributes according to current page table setup.
960 RefreshGcdMemoryAttributesFromPaging (
965 UINTN NumberOfDescriptors
;
966 EFI_GCD_MEMORY_SPACE_DESCRIPTOR
*MemorySpaceMap
;
967 PAGE_TABLE_LIB_PAGING_CONTEXT PagingContext
;
968 PAGE_ATTRIBUTE PageAttribute
;
971 UINT64 MemorySpaceLength
;
974 UINT64 PageStartAddress
;
977 UINT64 NewAttributes
;
981 // Assuming that memory space map returned is sorted already; otherwise sort
982 // them in the order of lowest address to highest address.
984 Status
= gDS
->GetMemorySpaceMap (&NumberOfDescriptors
, &MemorySpaceMap
);
985 ASSERT_EFI_ERROR (Status
);
987 GetCurrentPagingContext (&PagingContext
);
994 if (IsExecuteDisableEnabled ()) {
995 Capabilities
= EFI_MEMORY_RO
| EFI_MEMORY_RP
| EFI_MEMORY_XP
;
997 Capabilities
= EFI_MEMORY_RO
| EFI_MEMORY_RP
;
1000 for (Index
= 0; Index
< NumberOfDescriptors
; Index
++) {
1001 if (MemorySpaceMap
[Index
].GcdMemoryType
== EfiGcdMemoryTypeNonExistent
) {
1006 // Sync the actual paging related capabilities back to GCD service first.
1007 // As a side effect (good one), this can also help to avoid unnecessary
1008 // memory map entries due to the different capabilities of the same type
1009 // memory, such as multiple RT_CODE and RT_DATA entries in memory map,
1010 // which could cause boot failure of some old Linux distro (before v4.3).
1012 Status
= gDS
->SetMemorySpaceCapabilities (
1013 MemorySpaceMap
[Index
].BaseAddress
,
1014 MemorySpaceMap
[Index
].Length
,
1015 MemorySpaceMap
[Index
].Capabilities
| Capabilities
1017 if (EFI_ERROR (Status
)) {
1019 // If we cannot update the capabilities, we cannot update its
1020 // attributes either. So just simply skip current block of memory.
1024 "Failed to update capability: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
1026 MemorySpaceMap
[Index
].BaseAddress
,
1027 MemorySpaceMap
[Index
].BaseAddress
+ MemorySpaceMap
[Index
].Length
- 1,
1028 MemorySpaceMap
[Index
].Capabilities
,
1029 MemorySpaceMap
[Index
].Capabilities
| Capabilities
1034 if (MemorySpaceMap
[Index
].BaseAddress
>= (BaseAddress
+ PageLength
)) {
1036 // Current memory space starts at a new page. Resetting PageLength will
1037 // trigger a retrieval of page attributes at new address.
1042 // In case current memory space is not adjacent to last one
1044 PageLength
-= (MemorySpaceMap
[Index
].BaseAddress
- BaseAddress
);
1048 // Sync actual page attributes to GCD
1050 BaseAddress
= MemorySpaceMap
[Index
].BaseAddress
;
1051 MemorySpaceLength
= MemorySpaceMap
[Index
].Length
;
1052 while (MemorySpaceLength
> 0) {
1053 if (PageLength
== 0) {
1054 PageEntry
= GetPageTableEntry (&PagingContext
, BaseAddress
, &PageAttribute
);
1055 if (PageEntry
== NULL
) {
1060 // Note current memory space might start in the middle of a page
1062 PageStartAddress
= (*PageEntry
) & (UINT64
)PageAttributeToMask (PageAttribute
);
1063 PageLength
= PageAttributeToLength (PageAttribute
) - (BaseAddress
- PageStartAddress
);
1064 Attributes
= GetAttributesFromPageEntry (PageEntry
);
1067 Length
= MIN (PageLength
, MemorySpaceLength
);
1068 if (Attributes
!= (MemorySpaceMap
[Index
].Attributes
&
1069 EFI_MEMORY_ATTRIBUTE_MASK
))
1071 NewAttributes
= (MemorySpaceMap
[Index
].Attributes
&
1072 ~EFI_MEMORY_ATTRIBUTE_MASK
) | Attributes
;
1073 Status
= gDS
->SetMemorySpaceAttributes (
1078 ASSERT_EFI_ERROR (Status
);
1081 "Updated memory space attribute: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
1084 BaseAddress
+ Length
- 1,
1085 MemorySpaceMap
[Index
].Attributes
,
1090 PageLength
-= Length
;
1091 MemorySpaceLength
-= Length
;
1092 BaseAddress
+= Length
;
1096 FreePool (MemorySpaceMap
);
1100 Initialize a buffer pool for page table use only.
1102 To reduce the potential split operation on page table, the pages reserved for
1103 page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
1104 at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
1105 initialized with number of pages greater than or equal to the given PoolPages.
1107 Once the pages in the pool are used up, this method should be called again to
1108 reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. Usually this won't happen
1111 @param[in] PoolPages The least page number of the pool to be created.
1113 @retval TRUE The pool is initialized successfully.
1114 @retval FALSE The memory is out of resource.
1117 InitializePageTablePool (
1125 // Do not allow re-entrance.
1127 if (mPageTablePoolLock
) {
1131 mPageTablePoolLock
= TRUE
;
1135 // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
1138 PoolPages
+= 1; // Add one page for header.
1139 PoolPages
= ((PoolPages
- 1) / PAGE_TABLE_POOL_UNIT_PAGES
+ 1) *
1140 PAGE_TABLE_POOL_UNIT_PAGES
;
1141 Buffer
= AllocateAlignedPages (PoolPages
, PAGE_TABLE_POOL_ALIGNMENT
);
1142 if (Buffer
== NULL
) {
1143 DEBUG ((DEBUG_ERROR
, "ERROR: Out of aligned pages\r\n"));
1149 "Paging: added %lu pages to page table pool\r\n",
1154 // Link all pools into a list for easier track later.
1156 if (mPageTablePool
== NULL
) {
1157 mPageTablePool
= Buffer
;
1158 mPageTablePool
->NextPool
= mPageTablePool
;
1160 ((PAGE_TABLE_POOL
*)Buffer
)->NextPool
= mPageTablePool
->NextPool
;
1161 mPageTablePool
->NextPool
= Buffer
;
1162 mPageTablePool
= Buffer
;
1166 // Reserve one page for pool header.
1168 mPageTablePool
->FreePages
= PoolPages
- 1;
1169 mPageTablePool
->Offset
= EFI_PAGES_TO_SIZE (1);
1172 // Mark the whole pool pages as read-only.
1174 ConvertMemoryPageAttributes (
1176 (PHYSICAL_ADDRESS
)(UINTN
)Buffer
,
1177 EFI_PAGES_TO_SIZE (PoolPages
),
1180 AllocatePageTableMemory
,
1184 ASSERT (IsModified
== TRUE
);
1187 mPageTablePoolLock
= FALSE
;
1192 This API provides a way to allocate memory for page table.
1194 This API can be called more than once to allocate memory for page tables.
1196 Allocates the number of 4KB pages and returns a pointer to the allocated
1197 buffer. The buffer returned is aligned on a 4KB boundary.
1199 If Pages is 0, then NULL is returned.
1200 If there is not enough memory remaining to satisfy the request, then NULL is
1203 @param Pages The number of 4 KB pages to allocate.
1205 @return A pointer to the allocated buffer or NULL if allocation fails.
1210 AllocatePageTableMemory (
1221 // Renew the pool if necessary.
1223 if ((mPageTablePool
== NULL
) ||
1224 (Pages
> mPageTablePool
->FreePages
))
1226 if (!InitializePageTablePool (Pages
)) {
1231 Buffer
= (UINT8
*)mPageTablePool
+ mPageTablePool
->Offset
;
1233 mPageTablePool
->Offset
+= EFI_PAGES_TO_SIZE (Pages
);
1234 mPageTablePool
->FreePages
-= Pages
;
1240 Special handler for #DB exception, which will restore the page attributes
1241 (not-present). It should work with #PF handler which will set pages to
1244 @param ExceptionType Exception type.
1245 @param SystemContext Pointer to EFI_SYSTEM_CONTEXT.
1250 DebugExceptionHandler (
1251 IN EFI_EXCEPTION_TYPE ExceptionType
,
1252 IN EFI_SYSTEM_CONTEXT SystemContext
1257 BOOLEAN IsWpEnabled
;
1259 MpInitLibWhoAmI (&CpuIndex
);
1262 // Clear last PF entries
1264 IsWpEnabled
= IsReadOnlyPageWriteProtected ();
1266 DisableReadOnlyPageWriteProtect ();
1269 for (PFEntry
= 0; PFEntry
< mPFEntryCount
[CpuIndex
]; PFEntry
++) {
1270 if (mLastPFEntryPointer
[CpuIndex
][PFEntry
] != NULL
) {
1271 *mLastPFEntryPointer
[CpuIndex
][PFEntry
] &= ~(UINT64
)IA32_PG_P
;
1276 EnableReadOnlyPageWriteProtect ();
1280 // Reset page fault exception count for next page fault.
1282 mPFEntryCount
[CpuIndex
] = 0;
1290 // Clear TF in EFLAGS
1292 if (mPagingContext
.MachineType
== IMAGE_FILE_MACHINE_I386
) {
1293 SystemContext
.SystemContextIa32
->Eflags
&= (UINT32
) ~BIT8
;
1295 SystemContext
.SystemContextX64
->Rflags
&= (UINT64
) ~BIT8
;
1300 Special handler for #PF exception, which will set the pages which caused
1301 #PF to be 'present'. The attribute of those pages should be restored in
1302 the subsequent #DB handler.
1304 @param ExceptionType Exception type.
1305 @param SystemContext Pointer to EFI_SYSTEM_CONTEXT.
1310 PageFaultExceptionHandler (
1311 IN EFI_EXCEPTION_TYPE ExceptionType
,
1312 IN EFI_SYSTEM_CONTEXT SystemContext
1317 PAGE_TABLE_LIB_PAGING_CONTEXT PagingContext
;
1318 PAGE_ATTRIBUTE PageAttribute
;
1324 BOOLEAN NonStopMode
;
1326 PFAddress
= AsmReadCr2 () & ~EFI_PAGE_MASK
;
1327 if (PFAddress
< BASE_4KB
) {
1328 NonStopMode
= NULL_DETECTION_NONSTOP_MODE
? TRUE
: FALSE
;
1330 NonStopMode
= HEAP_GUARD_NONSTOP_MODE
? TRUE
: FALSE
;
1334 MpInitLibWhoAmI (&CpuIndex
);
1335 GetCurrentPagingContext (&PagingContext
);
1337 // Memory operation cross page boundary, like "rep mov" instruction, will
1338 // cause infinite loop between this and Debug Trap handler. We have to make
1339 // sure that current page and the page followed are both in PRESENT state.
1342 while (PageNumber
> 0) {
1343 PageEntry
= GetPageTableEntry (&PagingContext
, PFAddress
, &PageAttribute
);
1344 ASSERT (PageEntry
!= NULL
);
1346 if (PageEntry
!= NULL
) {
1347 Attributes
= GetAttributesFromPageEntry (PageEntry
);
1348 if ((Attributes
& EFI_MEMORY_RP
) != 0) {
1349 Attributes
&= ~EFI_MEMORY_RP
;
1350 Status
= AssignMemoryPageAttributes (
1357 if (!EFI_ERROR (Status
)) {
1358 Index
= mPFEntryCount
[CpuIndex
];
1360 // Re-retrieve page entry because above calling might update page
1361 // table due to table split.
1363 PageEntry
= GetPageTableEntry (&PagingContext
, PFAddress
, &PageAttribute
);
1364 mLastPFEntryPointer
[CpuIndex
][Index
++] = PageEntry
;
1365 mPFEntryCount
[CpuIndex
] = Index
;
1370 PFAddress
+= EFI_PAGE_SIZE
;
1376 // Initialize the serial port before dumping.
1378 SerialPortInitialize ();
1380 // Display ExceptionType, CPU information and Image information
1382 DumpCpuContext (ExceptionType
, SystemContext
);
1387 if (mPagingContext
.MachineType
== IMAGE_FILE_MACHINE_I386
) {
1388 SystemContext
.SystemContextIa32
->Eflags
|= (UINT32
)BIT8
;
1390 SystemContext
.SystemContextX64
->Rflags
|= (UINT64
)BIT8
;
1398 Initialize the Page Table lib.
1401 InitializePageTableLib (
1405 PAGE_TABLE_LIB_PAGING_CONTEXT CurrentPagingContext
;
1407 UINTN
*PageTableBase
;
1409 GetCurrentPagingContext (&CurrentPagingContext
);
1411 GetPagingDetails (&CurrentPagingContext
.ContextData
, &PageTableBase
, &Attributes
);
1414 // Reserve memory of page tables for future uses, if paging is enabled.
1416 if ((*PageTableBase
!= 0) &&
1417 ((*Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) != 0))
1419 DisableReadOnlyPageWriteProtect ();
1420 InitializePageTablePool (1);
1421 EnableReadOnlyPageWriteProtect ();
1424 if (HEAP_GUARD_NONSTOP_MODE
|| NULL_DETECTION_NONSTOP_MODE
) {
1425 mPFEntryCount
= (UINTN
*)AllocateZeroPool (sizeof (UINTN
) * mNumberOfProcessors
);
1426 ASSERT (mPFEntryCount
!= NULL
);
1428 mLastPFEntryPointer
= (UINT64
*(*)[MAX_PF_ENTRY_COUNT
])
1429 AllocateZeroPool (sizeof (mLastPFEntryPointer
[0]) * mNumberOfProcessors
);
1430 ASSERT (mLastPFEntryPointer
!= NULL
);
1433 DEBUG ((DEBUG_INFO
, "CurrentPagingContext:\n"));
1434 DEBUG ((DEBUG_INFO
, " MachineType - 0x%x\n", CurrentPagingContext
.MachineType
));
1435 DEBUG ((DEBUG_INFO
, " PageTableBase - 0x%Lx\n", (UINT64
)*PageTableBase
));
1436 DEBUG ((DEBUG_INFO
, " Attributes - 0x%x\n", *Attributes
));