2 Page table management support.
4 Copyright (c) 2017 - 2019, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
13 #include <Library/PeCoffGetEntryPointLib.h>
14 #include <Library/SerialPortLib.h>
15 #include <Library/SynchronizationLib.h>
16 #include <Library/PrintLib.h>
17 #include <Protocol/SmmBase2.h>
18 #include <Register/Cpuid.h>
19 #include <Register/Msr.h>
22 #include "CpuPageTable.h"
27 #define IA32_PG_P BIT0
28 #define IA32_PG_RW BIT1
29 #define IA32_PG_U BIT2
30 #define IA32_PG_WT BIT3
31 #define IA32_PG_CD BIT4
32 #define IA32_PG_A BIT5
33 #define IA32_PG_D BIT6
34 #define IA32_PG_PS BIT7
35 #define IA32_PG_PAT_2M BIT12
36 #define IA32_PG_PAT_4K IA32_PG_PS
37 #define IA32_PG_PMNT BIT62
38 #define IA32_PG_NX BIT63
40 #define PAGE_ATTRIBUTE_BITS (IA32_PG_D | IA32_PG_A | IA32_PG_U | IA32_PG_RW | IA32_PG_P)
42 // Bits 1, 2, 5, 6 are reserved in the IA32 PAE PDPTE
43 // X64 PAE PDPTE does not have such restriction
45 #define IA32_PAE_PDPTE_ATTRIBUTE_BITS (IA32_PG_P)
47 #define PAGE_PROGATE_BITS (IA32_PG_NX | PAGE_ATTRIBUTE_BITS)
49 #define PAGING_4K_MASK 0xFFF
50 #define PAGING_2M_MASK 0x1FFFFF
51 #define PAGING_1G_MASK 0x3FFFFFFF
53 #define PAGING_PAE_INDEX_MASK 0x1FF
55 #define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull
56 #define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull
57 #define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull
59 #define MAX_PF_ENTRY_COUNT 10
60 #define MAX_DEBUG_MESSAGE_LENGTH 0x100
61 #define IA32_PF_EC_ID BIT4
71 PAGE_ATTRIBUTE Attribute
;
74 } PAGE_ATTRIBUTE_TABLE
;
82 PAGE_ATTRIBUTE_TABLE mPageAttributeTable
[] = {
83 {Page4K
, SIZE_4KB
, PAGING_4K_ADDRESS_MASK_64
},
84 {Page2M
, SIZE_2MB
, PAGING_2M_ADDRESS_MASK_64
},
85 {Page1G
, SIZE_1GB
, PAGING_1G_ADDRESS_MASK_64
},
88 PAGE_TABLE_POOL
*mPageTablePool
= NULL
;
89 BOOLEAN mPageTablePoolLock
= FALSE
;
90 PAGE_TABLE_LIB_PAGING_CONTEXT mPagingContext
;
91 EFI_SMM_BASE2_PROTOCOL
*mSmmBase2
= NULL
;
94 // Record the page fault exception count for one instruction execution.
97 UINT64
*(*mLastPFEntryPointer
)[MAX_PF_ENTRY_COUNT
];
100 Check if current execution environment is in SMM mode or not, via
101 EFI_SMM_BASE2_PROTOCOL.
103 This is necessary because of the fact that MdePkg\Library\SmmMemoryAllocationLib
104 supports to free memory outside SMRAM. The library will call gBS->FreePool() or
105 gBS->FreePages() and then SetMemorySpaceAttributes interface in turn to change
106 memory paging attributes during free operation, if some memory related features
107 are enabled (like Heap Guard).
109 This means that SetMemorySpaceAttributes() has chance to run in SMM mode. This
110 will cause incorrect result because SMM mode always loads its own page tables,
111 which are usually different from DXE. This function can be used to detect such
112 situation and help to avoid further misoperations.
114 @retval TRUE In SMM mode.
115 @retval FALSE Not in SMM mode.
125 if (mSmmBase2
== NULL
) {
126 gBS
->LocateProtocol (&gEfiSmmBase2ProtocolGuid
, NULL
, (VOID
**)&mSmmBase2
);
129 if (mSmmBase2
!= NULL
) {
130 mSmmBase2
->InSmm (mSmmBase2
, &InSmm
);
134 // mSmmBase2->InSmm() can only detect if the caller is running in SMRAM
135 // or from SMM driver. It cannot tell if the caller is running in SMM mode.
136 // Check page table base address to guarantee that because SMM mode willl
137 // load its own page table.
140 mPagingContext
.ContextData
.X64
.PageTableBase
!= (UINT64
)AsmReadCr3());
144 Return current paging context.
146 @param[in,out] PagingContext The paging context.
149 GetCurrentPagingContext (
150 IN OUT PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
154 CPUID_EXTENDED_CPU_SIG_EDX RegEdx
;
155 MSR_IA32_EFER_REGISTER MsrEfer
;
160 // Don't retrieve current paging context from processor if in SMM mode.
163 ZeroMem (&mPagingContext
, sizeof(mPagingContext
));
164 if (sizeof(UINTN
) == sizeof(UINT64
)) {
165 mPagingContext
.MachineType
= IMAGE_FILE_MACHINE_X64
;
167 mPagingContext
.MachineType
= IMAGE_FILE_MACHINE_I386
;
170 Cr0
.UintN
= AsmReadCr0 ();
171 Cr4
.UintN
= AsmReadCr4 ();
173 if (Cr0
.Bits
.PG
!= 0) {
174 mPagingContext
.ContextData
.X64
.PageTableBase
= (AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64
);
176 mPagingContext
.ContextData
.X64
.PageTableBase
= 0;
178 if (Cr0
.Bits
.WP
!= 0) {
179 mPagingContext
.ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_WP_ENABLE
;
181 if (Cr4
.Bits
.PSE
!= 0) {
182 mPagingContext
.ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PSE
;
184 if (Cr4
.Bits
.PAE
!= 0) {
185 mPagingContext
.ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
;
188 AsmCpuid (CPUID_EXTENDED_FUNCTION
, &RegEax
, NULL
, NULL
, NULL
);
189 if (RegEax
>= CPUID_EXTENDED_CPU_SIG
) {
190 AsmCpuid (CPUID_EXTENDED_CPU_SIG
, NULL
, NULL
, NULL
, &RegEdx
.Uint32
);
192 if (RegEdx
.Bits
.NX
!= 0) {
194 MsrEfer
.Uint64
= AsmReadMsr64(MSR_CORE_IA32_EFER
);
195 if (MsrEfer
.Bits
.NXE
!= 0) {
197 mPagingContext
.ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED
;
201 if (RegEdx
.Bits
.Page1GB
!= 0) {
202 mPagingContext
.ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAGE_1G_SUPPORT
;
208 // This can avoid getting SMM paging context if in SMM mode. We cannot assume
209 // SMM mode shares the same paging context as DXE.
211 CopyMem (PagingContext
, &mPagingContext
, sizeof (mPagingContext
));
215 Return length according to page attributes.
217 @param[in] PageAttributes The page attribute of the page entry.
219 @return The length of page entry.
222 PageAttributeToLength (
223 IN PAGE_ATTRIBUTE PageAttribute
227 for (Index
= 0; Index
< sizeof(mPageAttributeTable
)/sizeof(mPageAttributeTable
[0]); Index
++) {
228 if (PageAttribute
== mPageAttributeTable
[Index
].Attribute
) {
229 return (UINTN
)mPageAttributeTable
[Index
].Length
;
236 Return address mask according to page attributes.
238 @param[in] PageAttributes The page attribute of the page entry.
240 @return The address mask of page entry.
243 PageAttributeToMask (
244 IN PAGE_ATTRIBUTE PageAttribute
248 for (Index
= 0; Index
< sizeof(mPageAttributeTable
)/sizeof(mPageAttributeTable
[0]); Index
++) {
249 if (PageAttribute
== mPageAttributeTable
[Index
].Attribute
) {
250 return (UINTN
)mPageAttributeTable
[Index
].AddressMask
;
257 Return page table entry to match the address.
259 @param[in] PagingContext The paging context.
260 @param[in] Address The address to be checked.
261 @param[out] PageAttributes The page attribute of the page entry.
263 @return The page entry.
267 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
,
268 IN PHYSICAL_ADDRESS Address
,
269 OUT PAGE_ATTRIBUTE
*PageAttribute
280 UINT64 AddressEncMask
;
282 ASSERT (PagingContext
!= NULL
);
284 Index4
= ((UINTN
)RShiftU64 (Address
, 39)) & PAGING_PAE_INDEX_MASK
;
285 Index3
= ((UINTN
)Address
>> 30) & PAGING_PAE_INDEX_MASK
;
286 Index2
= ((UINTN
)Address
>> 21) & PAGING_PAE_INDEX_MASK
;
287 Index1
= ((UINTN
)Address
>> 12) & PAGING_PAE_INDEX_MASK
;
289 // Make sure AddressEncMask is contained to smallest supported address field.
291 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
293 if (PagingContext
->MachineType
== IMAGE_FILE_MACHINE_X64
) {
294 L4PageTable
= (UINT64
*)(UINTN
)PagingContext
->ContextData
.X64
.PageTableBase
;
295 if (L4PageTable
[Index4
] == 0) {
296 *PageAttribute
= PageNone
;
300 L3PageTable
= (UINT64
*)(UINTN
)(L4PageTable
[Index4
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
302 ASSERT((PagingContext
->ContextData
.Ia32
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) != 0);
303 L3PageTable
= (UINT64
*)(UINTN
)PagingContext
->ContextData
.Ia32
.PageTableBase
;
305 if (L3PageTable
[Index3
] == 0) {
306 *PageAttribute
= PageNone
;
309 if ((L3PageTable
[Index3
] & IA32_PG_PS
) != 0) {
311 *PageAttribute
= Page1G
;
312 return &L3PageTable
[Index3
];
315 L2PageTable
= (UINT64
*)(UINTN
)(L3PageTable
[Index3
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
316 if (L2PageTable
[Index2
] == 0) {
317 *PageAttribute
= PageNone
;
320 if ((L2PageTable
[Index2
] & IA32_PG_PS
) != 0) {
322 *PageAttribute
= Page2M
;
323 return &L2PageTable
[Index2
];
327 L1PageTable
= (UINT64
*)(UINTN
)(L2PageTable
[Index2
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
328 if ((L1PageTable
[Index1
] == 0) && (Address
!= 0)) {
329 *PageAttribute
= PageNone
;
332 *PageAttribute
= Page4K
;
333 return &L1PageTable
[Index1
];
337 Return memory attributes of page entry.
339 @param[in] PageEntry The page entry.
341 @return Memory attributes of page entry.
344 GetAttributesFromPageEntry (
350 if ((*PageEntry
& IA32_PG_P
) == 0) {
351 Attributes
|= EFI_MEMORY_RP
;
353 if ((*PageEntry
& IA32_PG_RW
) == 0) {
354 Attributes
|= EFI_MEMORY_RO
;
356 if ((*PageEntry
& IA32_PG_NX
) != 0) {
357 Attributes
|= EFI_MEMORY_XP
;
363 Modify memory attributes of page entry.
365 @param[in] PagingContext The paging context.
366 @param[in] PageEntry The page entry.
367 @param[in] Attributes The bit mask of attributes to modify for the memory region.
368 @param[in] PageAction The page action.
369 @param[out] IsModified TRUE means page table modified. FALSE means page table not modified.
372 ConvertPageEntryAttribute (
373 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
,
374 IN UINT64
*PageEntry
,
375 IN UINT64 Attributes
,
376 IN PAGE_ACTION PageAction
,
377 OUT BOOLEAN
*IsModified
380 UINT64 CurrentPageEntry
;
383 CurrentPageEntry
= *PageEntry
;
384 NewPageEntry
= CurrentPageEntry
;
385 if ((Attributes
& EFI_MEMORY_RP
) != 0) {
386 switch (PageAction
) {
387 case PageActionAssign
:
389 NewPageEntry
&= ~(UINT64
)IA32_PG_P
;
391 case PageActionClear
:
392 NewPageEntry
|= IA32_PG_P
;
396 switch (PageAction
) {
397 case PageActionAssign
:
398 NewPageEntry
|= IA32_PG_P
;
401 case PageActionClear
:
405 if ((Attributes
& EFI_MEMORY_RO
) != 0) {
406 switch (PageAction
) {
407 case PageActionAssign
:
409 NewPageEntry
&= ~(UINT64
)IA32_PG_RW
;
411 case PageActionClear
:
412 NewPageEntry
|= IA32_PG_RW
;
416 switch (PageAction
) {
417 case PageActionAssign
:
418 NewPageEntry
|= IA32_PG_RW
;
421 case PageActionClear
:
425 if ((PagingContext
->ContextData
.Ia32
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED
) != 0) {
426 if ((Attributes
& EFI_MEMORY_XP
) != 0) {
427 switch (PageAction
) {
428 case PageActionAssign
:
430 NewPageEntry
|= IA32_PG_NX
;
432 case PageActionClear
:
433 NewPageEntry
&= ~IA32_PG_NX
;
437 switch (PageAction
) {
438 case PageActionAssign
:
439 NewPageEntry
&= ~IA32_PG_NX
;
442 case PageActionClear
:
447 *PageEntry
= NewPageEntry
;
448 if (CurrentPageEntry
!= NewPageEntry
) {
450 DEBUG ((DEBUG_VERBOSE
, "ConvertPageEntryAttribute 0x%lx", CurrentPageEntry
));
451 DEBUG ((DEBUG_VERBOSE
, "->0x%lx\n", NewPageEntry
));
458 This function returns if there is need to split page entry.
460 @param[in] BaseAddress The base address to be checked.
461 @param[in] Length The length to be checked.
462 @param[in] PageEntry The page entry to be checked.
463 @param[in] PageAttribute The page attribute of the page entry.
465 @retval SplitAttributes on if there is need to split page entry.
469 IN PHYSICAL_ADDRESS BaseAddress
,
471 IN UINT64
*PageEntry
,
472 IN PAGE_ATTRIBUTE PageAttribute
475 UINT64 PageEntryLength
;
477 PageEntryLength
= PageAttributeToLength (PageAttribute
);
479 if (((BaseAddress
& (PageEntryLength
- 1)) == 0) && (Length
>= PageEntryLength
)) {
483 if (((BaseAddress
& PAGING_2M_MASK
) != 0) || (Length
< SIZE_2MB
)) {
491 This function splits one page entry to small page entries.
493 @param[in] PageEntry The page entry to be splitted.
494 @param[in] PageAttribute The page attribute of the page entry.
495 @param[in] SplitAttribute How to split the page entry.
496 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
498 @retval RETURN_SUCCESS The page entry is splitted.
499 @retval RETURN_UNSUPPORTED The page entry does not support to be splitted.
500 @retval RETURN_OUT_OF_RESOURCES No resource to split page entry.
504 IN UINT64
*PageEntry
,
505 IN PAGE_ATTRIBUTE PageAttribute
,
506 IN PAGE_ATTRIBUTE SplitAttribute
,
507 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc
511 UINT64
*NewPageEntry
;
513 UINT64 AddressEncMask
;
515 ASSERT (PageAttribute
== Page2M
|| PageAttribute
== Page1G
);
517 ASSERT (AllocatePagesFunc
!= NULL
);
519 // Make sure AddressEncMask is contained to smallest supported address field.
521 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
523 if (PageAttribute
== Page2M
) {
527 ASSERT (SplitAttribute
== Page4K
);
528 if (SplitAttribute
== Page4K
) {
529 NewPageEntry
= AllocatePagesFunc (1);
530 DEBUG ((DEBUG_VERBOSE
, "Split - 0x%x\n", NewPageEntry
));
531 if (NewPageEntry
== NULL
) {
532 return RETURN_OUT_OF_RESOURCES
;
534 BaseAddress
= *PageEntry
& ~AddressEncMask
& PAGING_2M_ADDRESS_MASK_64
;
535 for (Index
= 0; Index
< SIZE_4KB
/ sizeof(UINT64
); Index
++) {
536 NewPageEntry
[Index
] = (BaseAddress
+ SIZE_4KB
* Index
) | AddressEncMask
| ((*PageEntry
) & PAGE_PROGATE_BITS
);
538 (*PageEntry
) = (UINT64
)(UINTN
)NewPageEntry
| AddressEncMask
| ((*PageEntry
) & PAGE_ATTRIBUTE_BITS
);
539 return RETURN_SUCCESS
;
541 return RETURN_UNSUPPORTED
;
543 } else if (PageAttribute
== Page1G
) {
546 // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
548 ASSERT (SplitAttribute
== Page2M
|| SplitAttribute
== Page4K
);
549 if ((SplitAttribute
== Page2M
|| SplitAttribute
== Page4K
)) {
550 NewPageEntry
= AllocatePagesFunc (1);
551 DEBUG ((DEBUG_VERBOSE
, "Split - 0x%x\n", NewPageEntry
));
552 if (NewPageEntry
== NULL
) {
553 return RETURN_OUT_OF_RESOURCES
;
555 BaseAddress
= *PageEntry
& ~AddressEncMask
& PAGING_1G_ADDRESS_MASK_64
;
556 for (Index
= 0; Index
< SIZE_4KB
/ sizeof(UINT64
); Index
++) {
557 NewPageEntry
[Index
] = (BaseAddress
+ SIZE_2MB
* Index
) | AddressEncMask
| IA32_PG_PS
| ((*PageEntry
) & PAGE_PROGATE_BITS
);
559 (*PageEntry
) = (UINT64
)(UINTN
)NewPageEntry
| AddressEncMask
| ((*PageEntry
) & PAGE_ATTRIBUTE_BITS
);
560 return RETURN_SUCCESS
;
562 return RETURN_UNSUPPORTED
;
565 return RETURN_UNSUPPORTED
;
570 Check the WP status in CR0 register. This bit is used to lock or unlock write
571 access to pages marked as read-only.
573 @retval TRUE Write protection is enabled.
574 @retval FALSE Write protection is disabled.
577 IsReadOnlyPageWriteProtected (
583 // To avoid unforseen consequences, don't touch paging settings in SMM mode
587 Cr0
.UintN
= AsmReadCr0 ();
588 return (BOOLEAN
) (Cr0
.Bits
.WP
!= 0);
594 Disable Write Protect on pages marked as read-only.
597 DisableReadOnlyPageWriteProtect (
603 // To avoid unforseen consequences, don't touch paging settings in SMM mode
607 Cr0
.UintN
= AsmReadCr0 ();
609 AsmWriteCr0 (Cr0
.UintN
);
614 Enable Write Protect on pages marked as read-only.
617 EnableReadOnlyPageWriteProtect (
623 // To avoid unforseen consequences, don't touch paging settings in SMM mode
627 Cr0
.UintN
= AsmReadCr0 ();
629 AsmWriteCr0 (Cr0
.UintN
);
634 This function modifies the page attributes for the memory region specified by BaseAddress and
635 Length from their current attributes to the attributes specified by Attributes.
637 Caller should make sure BaseAddress and Length is at page boundary.
639 @param[in] PagingContext The paging context. NULL means get page table from current CPU context.
640 @param[in] BaseAddress The physical address that is the start address of a memory region.
641 @param[in] Length The size in bytes of the memory region.
642 @param[in] Attributes The bit mask of attributes to modify for the memory region.
643 @param[in] PageAction The page action.
644 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
645 NULL mean page split is unsupported.
646 @param[out] IsSplitted TRUE means page table splitted. FALSE means page table not splitted.
647 @param[out] IsModified TRUE means page table modified. FALSE means page table not modified.
649 @retval RETURN_SUCCESS The attributes were modified for the memory region.
650 @retval RETURN_ACCESS_DENIED The attributes for the memory resource range specified by
651 BaseAddress and Length cannot be modified.
652 @retval RETURN_INVALID_PARAMETER Length is zero.
653 Attributes specified an illegal combination of attributes that
654 cannot be set together.
655 @retval RETURN_OUT_OF_RESOURCES There are not enough system resources to modify the attributes of
656 the memory resource range.
657 @retval RETURN_UNSUPPORTED The processor does not support one or more bytes of the memory
658 resource range specified by BaseAddress and Length.
659 The bit mask of attributes is not support for the memory resource
660 range specified by BaseAddress and Length.
663 ConvertMemoryPageAttributes (
664 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext OPTIONAL
,
665 IN PHYSICAL_ADDRESS BaseAddress
,
667 IN UINT64 Attributes
,
668 IN PAGE_ACTION PageAction
,
669 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc OPTIONAL
,
670 OUT BOOLEAN
*IsSplitted
, OPTIONAL
671 OUT BOOLEAN
*IsModified OPTIONAL
674 PAGE_TABLE_LIB_PAGING_CONTEXT CurrentPagingContext
;
676 PAGE_ATTRIBUTE PageAttribute
;
677 UINTN PageEntryLength
;
678 PAGE_ATTRIBUTE SplitAttribute
;
679 RETURN_STATUS Status
;
680 BOOLEAN IsEntryModified
;
683 if ((BaseAddress
& (SIZE_4KB
- 1)) != 0) {
684 DEBUG ((DEBUG_ERROR
, "BaseAddress(0x%lx) is not aligned!\n", BaseAddress
));
685 return EFI_UNSUPPORTED
;
687 if ((Length
& (SIZE_4KB
- 1)) != 0) {
688 DEBUG ((DEBUG_ERROR
, "Length(0x%lx) is not aligned!\n", Length
));
689 return EFI_UNSUPPORTED
;
692 DEBUG ((DEBUG_ERROR
, "Length is 0!\n"));
693 return RETURN_INVALID_PARAMETER
;
696 if ((Attributes
& ~(EFI_MEMORY_RP
| EFI_MEMORY_RO
| EFI_MEMORY_XP
)) != 0) {
697 DEBUG ((DEBUG_ERROR
, "Attributes(0x%lx) has unsupported bit\n", Attributes
));
698 return EFI_UNSUPPORTED
;
701 if (PagingContext
== NULL
) {
702 GetCurrentPagingContext (&CurrentPagingContext
);
704 CopyMem (&CurrentPagingContext
, PagingContext
, sizeof(CurrentPagingContext
));
706 switch(CurrentPagingContext
.MachineType
) {
707 case IMAGE_FILE_MACHINE_I386
:
708 if (CurrentPagingContext
.ContextData
.Ia32
.PageTableBase
== 0) {
709 if (Attributes
== 0) {
712 DEBUG ((DEBUG_ERROR
, "PageTable is 0!\n"));
713 return EFI_UNSUPPORTED
;
716 if ((CurrentPagingContext
.ContextData
.Ia32
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) == 0) {
717 DEBUG ((DEBUG_ERROR
, "Non-PAE Paging!\n"));
718 return EFI_UNSUPPORTED
;
720 if ((BaseAddress
+ Length
) > BASE_4GB
) {
721 DEBUG ((DEBUG_ERROR
, "Beyond 4GB memory in 32-bit mode!\n"));
722 return EFI_UNSUPPORTED
;
725 case IMAGE_FILE_MACHINE_X64
:
726 ASSERT (CurrentPagingContext
.ContextData
.X64
.PageTableBase
!= 0);
730 return EFI_UNSUPPORTED
;
734 // DEBUG ((DEBUG_ERROR, "ConvertMemoryPageAttributes(%x) - %016lx, %016lx, %02lx\n", IsSet, BaseAddress, Length, Attributes));
736 if (IsSplitted
!= NULL
) {
739 if (IsModified
!= NULL
) {
742 if (AllocatePagesFunc
== NULL
) {
743 AllocatePagesFunc
= AllocatePageTableMemory
;
747 // Make sure that the page table is changeable.
749 IsWpEnabled
= IsReadOnlyPageWriteProtected ();
751 DisableReadOnlyPageWriteProtect ();
755 // Below logic is to check 2M/4K page to make sure we donot waist memory.
757 Status
= EFI_SUCCESS
;
758 while (Length
!= 0) {
759 PageEntry
= GetPageTableEntry (&CurrentPagingContext
, BaseAddress
, &PageAttribute
);
760 if (PageEntry
== NULL
) {
761 Status
= RETURN_UNSUPPORTED
;
764 PageEntryLength
= PageAttributeToLength (PageAttribute
);
765 SplitAttribute
= NeedSplitPage (BaseAddress
, Length
, PageEntry
, PageAttribute
);
766 if (SplitAttribute
== PageNone
) {
767 ConvertPageEntryAttribute (&CurrentPagingContext
, PageEntry
, Attributes
, PageAction
, &IsEntryModified
);
768 if (IsEntryModified
) {
769 if (IsModified
!= NULL
) {
774 // Convert success, move to next
776 BaseAddress
+= PageEntryLength
;
777 Length
-= PageEntryLength
;
779 if (AllocatePagesFunc
== NULL
) {
780 Status
= RETURN_UNSUPPORTED
;
783 Status
= SplitPage (PageEntry
, PageAttribute
, SplitAttribute
, AllocatePagesFunc
);
784 if (RETURN_ERROR (Status
)) {
785 Status
= RETURN_UNSUPPORTED
;
788 if (IsSplitted
!= NULL
) {
791 if (IsModified
!= NULL
) {
795 // Just split current page
796 // Convert success in next around
803 // Restore page table write protection, if any.
806 EnableReadOnlyPageWriteProtect ();
812 This function assigns the page attributes for the memory region specified by BaseAddress and
813 Length from their current attributes to the attributes specified by Attributes.
815 Caller should make sure BaseAddress and Length is at page boundary.
817 Caller need guarentee the TPL <= TPL_NOTIFY, if there is split page request.
819 @param[in] PagingContext The paging context. NULL means get page table from current CPU context.
820 @param[in] BaseAddress The physical address that is the start address of a memory region.
821 @param[in] Length The size in bytes of the memory region.
822 @param[in] Attributes The bit mask of attributes to set for the memory region.
823 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
824 NULL mean page split is unsupported.
826 @retval RETURN_SUCCESS The attributes were cleared for the memory region.
827 @retval RETURN_ACCESS_DENIED The attributes for the memory resource range specified by
828 BaseAddress and Length cannot be modified.
829 @retval RETURN_INVALID_PARAMETER Length is zero.
830 Attributes specified an illegal combination of attributes that
831 cannot be set together.
832 @retval RETURN_OUT_OF_RESOURCES There are not enough system resources to modify the attributes of
833 the memory resource range.
834 @retval RETURN_UNSUPPORTED The processor does not support one or more bytes of the memory
835 resource range specified by BaseAddress and Length.
836 The bit mask of attributes is not support for the memory resource
837 range specified by BaseAddress and Length.
841 AssignMemoryPageAttributes (
842 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext OPTIONAL
,
843 IN PHYSICAL_ADDRESS BaseAddress
,
845 IN UINT64 Attributes
,
846 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc OPTIONAL
849 RETURN_STATUS Status
;
853 // DEBUG((DEBUG_INFO, "AssignMemoryPageAttributes: 0x%lx - 0x%lx (0x%lx)\n", BaseAddress, Length, Attributes));
854 Status
= ConvertMemoryPageAttributes (PagingContext
, BaseAddress
, Length
, Attributes
, PageActionAssign
, AllocatePagesFunc
, &IsSplitted
, &IsModified
);
855 if (!EFI_ERROR(Status
)) {
856 if ((PagingContext
== NULL
) && IsModified
) {
858 // Flush TLB as last step.
860 // Note: Since APs will always init CR3 register in HLT loop mode or do
861 // TLB flush in MWAIT loop mode, there's no need to flush TLB for them
872 Check if Execute Disable feature is enabled or not.
875 IsExecuteDisableEnabled (
879 MSR_CORE_IA32_EFER_REGISTER MsrEfer
;
881 MsrEfer
.Uint64
= AsmReadMsr64 (MSR_IA32_EFER
);
882 return (MsrEfer
.Bits
.NXE
== 1);
886 Update GCD memory space attributes according to current page table setup.
889 RefreshGcdMemoryAttributesFromPaging (
894 UINTN NumberOfDescriptors
;
895 EFI_GCD_MEMORY_SPACE_DESCRIPTOR
*MemorySpaceMap
;
896 PAGE_TABLE_LIB_PAGING_CONTEXT PagingContext
;
897 PAGE_ATTRIBUTE PageAttribute
;
900 UINT64 MemorySpaceLength
;
903 UINT64 PageStartAddress
;
906 UINT64 NewAttributes
;
910 // Assuming that memory space map returned is sorted already; otherwise sort
911 // them in the order of lowest address to highest address.
913 Status
= gDS
->GetMemorySpaceMap (&NumberOfDescriptors
, &MemorySpaceMap
);
914 ASSERT_EFI_ERROR (Status
);
916 GetCurrentPagingContext (&PagingContext
);
923 if (IsExecuteDisableEnabled ()) {
924 Capabilities
= EFI_MEMORY_RO
| EFI_MEMORY_RP
| EFI_MEMORY_XP
;
926 Capabilities
= EFI_MEMORY_RO
| EFI_MEMORY_RP
;
929 for (Index
= 0; Index
< NumberOfDescriptors
; Index
++) {
930 if (MemorySpaceMap
[Index
].GcdMemoryType
== EfiGcdMemoryTypeNonExistent
) {
935 // Sync the actual paging related capabilities back to GCD service first.
936 // As a side effect (good one), this can also help to avoid unnecessary
937 // memory map entries due to the different capabilities of the same type
938 // memory, such as multiple RT_CODE and RT_DATA entries in memory map,
939 // which could cause boot failure of some old Linux distro (before v4.3).
941 Status
= gDS
->SetMemorySpaceCapabilities (
942 MemorySpaceMap
[Index
].BaseAddress
,
943 MemorySpaceMap
[Index
].Length
,
944 MemorySpaceMap
[Index
].Capabilities
| Capabilities
946 if (EFI_ERROR (Status
)) {
948 // If we cannot udpate the capabilities, we cannot update its
949 // attributes either. So just simply skip current block of memory.
953 "Failed to update capability: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
954 (UINT64
)Index
, MemorySpaceMap
[Index
].BaseAddress
,
955 MemorySpaceMap
[Index
].BaseAddress
+ MemorySpaceMap
[Index
].Length
- 1,
956 MemorySpaceMap
[Index
].Capabilities
,
957 MemorySpaceMap
[Index
].Capabilities
| Capabilities
962 if (MemorySpaceMap
[Index
].BaseAddress
>= (BaseAddress
+ PageLength
)) {
964 // Current memory space starts at a new page. Resetting PageLength will
965 // trigger a retrieval of page attributes at new address.
970 // In case current memory space is not adjacent to last one
972 PageLength
-= (MemorySpaceMap
[Index
].BaseAddress
- BaseAddress
);
976 // Sync actual page attributes to GCD
978 BaseAddress
= MemorySpaceMap
[Index
].BaseAddress
;
979 MemorySpaceLength
= MemorySpaceMap
[Index
].Length
;
980 while (MemorySpaceLength
> 0) {
981 if (PageLength
== 0) {
982 PageEntry
= GetPageTableEntry (&PagingContext
, BaseAddress
, &PageAttribute
);
983 if (PageEntry
== NULL
) {
988 // Note current memory space might start in the middle of a page
990 PageStartAddress
= (*PageEntry
) & (UINT64
)PageAttributeToMask(PageAttribute
);
991 PageLength
= PageAttributeToLength (PageAttribute
) - (BaseAddress
- PageStartAddress
);
992 Attributes
= GetAttributesFromPageEntry (PageEntry
);
995 Length
= MIN (PageLength
, MemorySpaceLength
);
996 if (Attributes
!= (MemorySpaceMap
[Index
].Attributes
&
997 EFI_MEMORY_PAGETYPE_MASK
)) {
998 NewAttributes
= (MemorySpaceMap
[Index
].Attributes
&
999 ~EFI_MEMORY_PAGETYPE_MASK
) | Attributes
;
1000 Status
= gDS
->SetMemorySpaceAttributes (
1005 ASSERT_EFI_ERROR (Status
);
1008 "Updated memory space attribute: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
1009 (UINT64
)Index
, BaseAddress
, BaseAddress
+ Length
- 1,
1010 MemorySpaceMap
[Index
].Attributes
,
1015 PageLength
-= Length
;
1016 MemorySpaceLength
-= Length
;
1017 BaseAddress
+= Length
;
1021 FreePool (MemorySpaceMap
);
1025 Initialize a buffer pool for page table use only.
1027 To reduce the potential split operation on page table, the pages reserved for
1028 page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
1029 at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
1030 initialized with number of pages greater than or equal to the given PoolPages.
1032 Once the pages in the pool are used up, this method should be called again to
1033 reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. Usually this won't happen
1036 @param[in] PoolPages The least page number of the pool to be created.
1038 @retval TRUE The pool is initialized successfully.
1039 @retval FALSE The memory is out of resource.
1042 InitializePageTablePool (
1050 // Do not allow re-entrance.
1052 if (mPageTablePoolLock
) {
1056 mPageTablePoolLock
= TRUE
;
1060 // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
1063 PoolPages
+= 1; // Add one page for header.
1064 PoolPages
= ((PoolPages
- 1) / PAGE_TABLE_POOL_UNIT_PAGES
+ 1) *
1065 PAGE_TABLE_POOL_UNIT_PAGES
;
1066 Buffer
= AllocateAlignedPages (PoolPages
, PAGE_TABLE_POOL_ALIGNMENT
);
1067 if (Buffer
== NULL
) {
1068 DEBUG ((DEBUG_ERROR
, "ERROR: Out of aligned pages\r\n"));
1074 "Paging: added %lu pages to page table pool\r\n",
1079 // Link all pools into a list for easier track later.
1081 if (mPageTablePool
== NULL
) {
1082 mPageTablePool
= Buffer
;
1083 mPageTablePool
->NextPool
= mPageTablePool
;
1085 ((PAGE_TABLE_POOL
*)Buffer
)->NextPool
= mPageTablePool
->NextPool
;
1086 mPageTablePool
->NextPool
= Buffer
;
1087 mPageTablePool
= Buffer
;
1091 // Reserve one page for pool header.
1093 mPageTablePool
->FreePages
= PoolPages
- 1;
1094 mPageTablePool
->Offset
= EFI_PAGES_TO_SIZE (1);
1097 // Mark the whole pool pages as read-only.
1099 ConvertMemoryPageAttributes (
1101 (PHYSICAL_ADDRESS
)(UINTN
)Buffer
,
1102 EFI_PAGES_TO_SIZE (PoolPages
),
1105 AllocatePageTableMemory
,
1109 ASSERT (IsModified
== TRUE
);
1112 mPageTablePoolLock
= FALSE
;
1117 This API provides a way to allocate memory for page table.
1119 This API can be called more than once to allocate memory for page tables.
1121 Allocates the number of 4KB pages and returns a pointer to the allocated
1122 buffer. The buffer returned is aligned on a 4KB boundary.
1124 If Pages is 0, then NULL is returned.
1125 If there is not enough memory remaining to satisfy the request, then NULL is
1128 @param Pages The number of 4 KB pages to allocate.
1130 @return A pointer to the allocated buffer or NULL if allocation fails.
1135 AllocatePageTableMemory (
1146 // Renew the pool if necessary.
1148 if (mPageTablePool
== NULL
||
1149 Pages
> mPageTablePool
->FreePages
) {
1150 if (!InitializePageTablePool (Pages
)) {
1155 Buffer
= (UINT8
*)mPageTablePool
+ mPageTablePool
->Offset
;
1157 mPageTablePool
->Offset
+= EFI_PAGES_TO_SIZE (Pages
);
1158 mPageTablePool
->FreePages
-= Pages
;
1164 Special handler for #DB exception, which will restore the page attributes
1165 (not-present). It should work with #PF handler which will set pages to
1168 @param ExceptionType Exception type.
1169 @param SystemContext Pointer to EFI_SYSTEM_CONTEXT.
1174 DebugExceptionHandler (
1175 IN EFI_EXCEPTION_TYPE ExceptionType
,
1176 IN EFI_SYSTEM_CONTEXT SystemContext
1181 BOOLEAN IsWpEnabled
;
1183 MpInitLibWhoAmI (&CpuIndex
);
1186 // Clear last PF entries
1188 IsWpEnabled
= IsReadOnlyPageWriteProtected ();
1190 DisableReadOnlyPageWriteProtect ();
1193 for (PFEntry
= 0; PFEntry
< mPFEntryCount
[CpuIndex
]; PFEntry
++) {
1194 if (mLastPFEntryPointer
[CpuIndex
][PFEntry
] != NULL
) {
1195 *mLastPFEntryPointer
[CpuIndex
][PFEntry
] &= ~(UINT64
)IA32_PG_P
;
1200 EnableReadOnlyPageWriteProtect ();
1204 // Reset page fault exception count for next page fault.
1206 mPFEntryCount
[CpuIndex
] = 0;
1214 // Clear TF in EFLAGS
1216 if (mPagingContext
.MachineType
== IMAGE_FILE_MACHINE_I386
) {
1217 SystemContext
.SystemContextIa32
->Eflags
&= (UINT32
)~BIT8
;
1219 SystemContext
.SystemContextX64
->Rflags
&= (UINT64
)~BIT8
;
1224 Special handler for #PF exception, which will set the pages which caused
1225 #PF to be 'present'. The attribute of those pages should be restored in
1226 the subsequent #DB handler.
1228 @param ExceptionType Exception type.
1229 @param SystemContext Pointer to EFI_SYSTEM_CONTEXT.
1234 PageFaultExceptionHandler (
1235 IN EFI_EXCEPTION_TYPE ExceptionType
,
1236 IN EFI_SYSTEM_CONTEXT SystemContext
1241 PAGE_TABLE_LIB_PAGING_CONTEXT PagingContext
;
1242 PAGE_ATTRIBUTE PageAttribute
;
1248 BOOLEAN NonStopMode
;
1250 PFAddress
= AsmReadCr2 () & ~EFI_PAGE_MASK
;
1251 if (PFAddress
< BASE_4KB
) {
1252 NonStopMode
= NULL_DETECTION_NONSTOP_MODE
? TRUE
: FALSE
;
1254 NonStopMode
= HEAP_GUARD_NONSTOP_MODE
? TRUE
: FALSE
;
1258 MpInitLibWhoAmI (&CpuIndex
);
1259 GetCurrentPagingContext (&PagingContext
);
1261 // Memory operation cross page boundary, like "rep mov" instruction, will
1262 // cause infinite loop between this and Debug Trap handler. We have to make
1263 // sure that current page and the page followed are both in PRESENT state.
1266 while (PageNumber
> 0) {
1267 PageEntry
= GetPageTableEntry (&PagingContext
, PFAddress
, &PageAttribute
);
1268 ASSERT(PageEntry
!= NULL
);
1270 if (PageEntry
!= NULL
) {
1271 Attributes
= GetAttributesFromPageEntry (PageEntry
);
1272 if ((Attributes
& EFI_MEMORY_RP
) != 0) {
1273 Attributes
&= ~EFI_MEMORY_RP
;
1274 Status
= AssignMemoryPageAttributes (&PagingContext
, PFAddress
,
1275 EFI_PAGE_SIZE
, Attributes
, NULL
);
1276 if (!EFI_ERROR(Status
)) {
1277 Index
= mPFEntryCount
[CpuIndex
];
1279 // Re-retrieve page entry because above calling might update page
1280 // table due to table split.
1282 PageEntry
= GetPageTableEntry (&PagingContext
, PFAddress
, &PageAttribute
);
1283 mLastPFEntryPointer
[CpuIndex
][Index
++] = PageEntry
;
1284 mPFEntryCount
[CpuIndex
] = Index
;
1289 PFAddress
+= EFI_PAGE_SIZE
;
1295 // Initialize the serial port before dumping.
1297 SerialPortInitialize ();
1299 // Display ExceptionType, CPU information and Image information
1301 DumpCpuContext (ExceptionType
, SystemContext
);
1306 if (mPagingContext
.MachineType
== IMAGE_FILE_MACHINE_I386
) {
1307 SystemContext
.SystemContextIa32
->Eflags
|= (UINT32
)BIT8
;
1309 SystemContext
.SystemContextX64
->Rflags
|= (UINT64
)BIT8
;
1317 Initialize the Page Table lib.
1320 InitializePageTableLib (
1324 PAGE_TABLE_LIB_PAGING_CONTEXT CurrentPagingContext
;
1326 GetCurrentPagingContext (&CurrentPagingContext
);
1329 // Reserve memory of page tables for future uses, if paging is enabled.
1331 if (CurrentPagingContext
.ContextData
.X64
.PageTableBase
!= 0 &&
1332 (CurrentPagingContext
.ContextData
.Ia32
.Attributes
&
1333 PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) != 0) {
1334 DisableReadOnlyPageWriteProtect ();
1335 InitializePageTablePool (1);
1336 EnableReadOnlyPageWriteProtect ();
1339 if (HEAP_GUARD_NONSTOP_MODE
|| NULL_DETECTION_NONSTOP_MODE
) {
1340 mPFEntryCount
= (UINTN
*)AllocateZeroPool (sizeof (UINTN
) * mNumberOfProcessors
);
1341 ASSERT (mPFEntryCount
!= NULL
);
1343 mLastPFEntryPointer
= (UINT64
*(*)[MAX_PF_ENTRY_COUNT
])
1344 AllocateZeroPool (sizeof (mLastPFEntryPointer
[0]) * mNumberOfProcessors
);
1345 ASSERT (mLastPFEntryPointer
!= NULL
);
1348 DEBUG ((DEBUG_INFO
, "CurrentPagingContext:\n", CurrentPagingContext
.MachineType
));
1349 DEBUG ((DEBUG_INFO
, " MachineType - 0x%x\n", CurrentPagingContext
.MachineType
));
1350 DEBUG ((DEBUG_INFO
, " PageTableBase - 0x%x\n", CurrentPagingContext
.ContextData
.X64
.PageTableBase
));
1351 DEBUG ((DEBUG_INFO
, " Attributes - 0x%x\n", CurrentPagingContext
.ContextData
.X64
.Attributes
));