2 Page table management support.
4 Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
7 This program and the accompanying materials
8 are licensed and made available under the terms and conditions of the BSD License
9 which accompanies this distribution. The full text of the license may be found at
10 http://opensource.org/licenses/bsd-license.php
12 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
13 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
19 #include <Library/PeCoffGetEntryPointLib.h>
20 #include <Library/SerialPortLib.h>
21 #include <Library/SynchronizationLib.h>
22 #include <Library/PrintLib.h>
23 #include <Protocol/SmmBase2.h>
24 #include <Register/Cpuid.h>
25 #include <Register/Msr.h>
28 #include "CpuPageTable.h"
41 #define IA32_PG_P BIT0
42 #define IA32_PG_RW BIT1
43 #define IA32_PG_U BIT2
44 #define IA32_PG_WT BIT3
45 #define IA32_PG_CD BIT4
46 #define IA32_PG_A BIT5
47 #define IA32_PG_D BIT6
48 #define IA32_PG_PS BIT7
49 #define IA32_PG_PAT_2M BIT12
50 #define IA32_PG_PAT_4K IA32_PG_PS
51 #define IA32_PG_PMNT BIT62
52 #define IA32_PG_NX BIT63
54 #define PAGE_ATTRIBUTE_BITS (IA32_PG_D | IA32_PG_A | IA32_PG_U | IA32_PG_RW | IA32_PG_P)
56 // Bits 1, 2, 5, 6 are reserved in the IA32 PAE PDPTE
57 // X64 PAE PDPTE does not have such restriction
59 #define IA32_PAE_PDPTE_ATTRIBUTE_BITS (IA32_PG_P)
61 #define PAGE_PROGATE_BITS (IA32_PG_NX | PAGE_ATTRIBUTE_BITS)
63 #define PAGING_4K_MASK 0xFFF
64 #define PAGING_2M_MASK 0x1FFFFF
65 #define PAGING_1G_MASK 0x3FFFFFFF
67 #define PAGING_PAE_INDEX_MASK 0x1FF
69 #define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull
70 #define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull
71 #define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull
73 #define MAX_PF_ENTRY_COUNT 10
74 #define MAX_DEBUG_MESSAGE_LENGTH 0x100
75 #define IA32_PF_EC_ID BIT4
85 PAGE_ATTRIBUTE Attribute
;
88 } PAGE_ATTRIBUTE_TABLE
;
96 PAGE_ATTRIBUTE_TABLE mPageAttributeTable
[] = {
97 {Page4K
, SIZE_4KB
, PAGING_4K_ADDRESS_MASK_64
},
98 {Page2M
, SIZE_2MB
, PAGING_2M_ADDRESS_MASK_64
},
99 {Page1G
, SIZE_1GB
, PAGING_1G_ADDRESS_MASK_64
},
102 PAGE_TABLE_POOL
*mPageTablePool
= NULL
;
103 BOOLEAN mPageTablePoolLock
= FALSE
;
104 PAGE_TABLE_LIB_PAGING_CONTEXT mPagingContext
;
105 EFI_SMM_BASE2_PROTOCOL
*mSmmBase2
= NULL
;
108 // Record the page fault exception count for one instruction execution.
110 UINTN
*mPFEntryCount
;
111 UINT64
*(*mLastPFEntryPointer
)[MAX_PF_ENTRY_COUNT
];
114 Check if current execution environment is in SMM mode or not, via
115 EFI_SMM_BASE2_PROTOCOL.
117 This is necessary because of the fact that MdePkg\Library\SmmMemoryAllocationLib
118 supports to free memory outside SMRAM. The library will call gBS->FreePool() or
119 gBS->FreePages() and then SetMemorySpaceAttributes interface in turn to change
120 memory paging attributes during free operation, if some memory related features
121 are enabled (like Heap Guard).
123 This means that SetMemorySpaceAttributes() has chance to run in SMM mode. This
124 will cause incorrect result because SMM mode always loads its own page tables,
125 which are usually different from DXE. This function can be used to detect such
126 situation and help to avoid further misoperations.
128 @retval TRUE In SMM mode.
129 @retval FALSE Not in SMM mode.
139 if (mSmmBase2
== NULL
) {
140 gBS
->LocateProtocol (&gEfiSmmBase2ProtocolGuid
, NULL
, (VOID
**)&mSmmBase2
);
143 if (mSmmBase2
!= NULL
) {
144 mSmmBase2
->InSmm (mSmmBase2
, &InSmm
);
148 // mSmmBase2->InSmm() can only detect if the caller is running in SMRAM
149 // or from SMM driver. It cannot tell if the caller is running in SMM mode.
150 // Check page table base address to guarantee that because SMM mode willl
151 // load its own page table.
154 mPagingContext
.ContextData
.X64
.PageTableBase
!= (UINT64
)AsmReadCr3());
158 Return current paging context.
160 @param[in,out] PagingContext The paging context.
163 GetCurrentPagingContext (
164 IN OUT PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
168 CPUID_EXTENDED_CPU_SIG_EDX RegEdx
;
169 MSR_IA32_EFER_REGISTER MsrEfer
;
172 // Don't retrieve current paging context from processor if in SMM mode.
175 ZeroMem (&mPagingContext
, sizeof(mPagingContext
));
176 if (sizeof(UINTN
) == sizeof(UINT64
)) {
177 mPagingContext
.MachineType
= IMAGE_FILE_MACHINE_X64
;
179 mPagingContext
.MachineType
= IMAGE_FILE_MACHINE_I386
;
181 if ((AsmReadCr0 () & CR0_PG
) != 0) {
182 mPagingContext
.ContextData
.X64
.PageTableBase
= (AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64
);
184 mPagingContext
.ContextData
.X64
.PageTableBase
= 0;
187 if ((AsmReadCr4 () & CR4_PSE
) != 0) {
188 mPagingContext
.ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PSE
;
190 if ((AsmReadCr4 () & CR4_PAE
) != 0) {
191 mPagingContext
.ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
;
193 if ((AsmReadCr0 () & CR0_WP
) != 0) {
194 mPagingContext
.ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_WP_ENABLE
;
197 AsmCpuid (CPUID_EXTENDED_FUNCTION
, &RegEax
, NULL
, NULL
, NULL
);
198 if (RegEax
>= CPUID_EXTENDED_CPU_SIG
) {
199 AsmCpuid (CPUID_EXTENDED_CPU_SIG
, NULL
, NULL
, NULL
, &RegEdx
.Uint32
);
201 if (RegEdx
.Bits
.NX
!= 0) {
203 MsrEfer
.Uint64
= AsmReadMsr64(MSR_CORE_IA32_EFER
);
204 if (MsrEfer
.Bits
.NXE
!= 0) {
206 mPagingContext
.ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED
;
210 if (RegEdx
.Bits
.Page1GB
!= 0) {
211 mPagingContext
.ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAGE_1G_SUPPORT
;
217 // This can avoid getting SMM paging context if in SMM mode. We cannot assume
218 // SMM mode shares the same paging context as DXE.
220 CopyMem (PagingContext
, &mPagingContext
, sizeof (mPagingContext
));
224 Return length according to page attributes.
226 @param[in] PageAttributes The page attribute of the page entry.
228 @return The length of page entry.
231 PageAttributeToLength (
232 IN PAGE_ATTRIBUTE PageAttribute
236 for (Index
= 0; Index
< sizeof(mPageAttributeTable
)/sizeof(mPageAttributeTable
[0]); Index
++) {
237 if (PageAttribute
== mPageAttributeTable
[Index
].Attribute
) {
238 return (UINTN
)mPageAttributeTable
[Index
].Length
;
245 Return address mask according to page attributes.
247 @param[in] PageAttributes The page attribute of the page entry.
249 @return The address mask of page entry.
252 PageAttributeToMask (
253 IN PAGE_ATTRIBUTE PageAttribute
257 for (Index
= 0; Index
< sizeof(mPageAttributeTable
)/sizeof(mPageAttributeTable
[0]); Index
++) {
258 if (PageAttribute
== mPageAttributeTable
[Index
].Attribute
) {
259 return (UINTN
)mPageAttributeTable
[Index
].AddressMask
;
266 Return page table entry to match the address.
268 @param[in] PagingContext The paging context.
269 @param[in] Address The address to be checked.
270 @param[out] PageAttributes The page attribute of the page entry.
272 @return The page entry.
276 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
,
277 IN PHYSICAL_ADDRESS Address
,
278 OUT PAGE_ATTRIBUTE
*PageAttribute
289 UINT64 AddressEncMask
;
291 ASSERT (PagingContext
!= NULL
);
293 Index4
= ((UINTN
)RShiftU64 (Address
, 39)) & PAGING_PAE_INDEX_MASK
;
294 Index3
= ((UINTN
)Address
>> 30) & PAGING_PAE_INDEX_MASK
;
295 Index2
= ((UINTN
)Address
>> 21) & PAGING_PAE_INDEX_MASK
;
296 Index1
= ((UINTN
)Address
>> 12) & PAGING_PAE_INDEX_MASK
;
298 // Make sure AddressEncMask is contained to smallest supported address field.
300 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
302 if (PagingContext
->MachineType
== IMAGE_FILE_MACHINE_X64
) {
303 L4PageTable
= (UINT64
*)(UINTN
)PagingContext
->ContextData
.X64
.PageTableBase
;
304 if (L4PageTable
[Index4
] == 0) {
305 *PageAttribute
= PageNone
;
309 L3PageTable
= (UINT64
*)(UINTN
)(L4PageTable
[Index4
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
311 ASSERT((PagingContext
->ContextData
.Ia32
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) != 0);
312 L3PageTable
= (UINT64
*)(UINTN
)PagingContext
->ContextData
.Ia32
.PageTableBase
;
314 if (L3PageTable
[Index3
] == 0) {
315 *PageAttribute
= PageNone
;
318 if ((L3PageTable
[Index3
] & IA32_PG_PS
) != 0) {
320 *PageAttribute
= Page1G
;
321 return &L3PageTable
[Index3
];
324 L2PageTable
= (UINT64
*)(UINTN
)(L3PageTable
[Index3
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
325 if (L2PageTable
[Index2
] == 0) {
326 *PageAttribute
= PageNone
;
329 if ((L2PageTable
[Index2
] & IA32_PG_PS
) != 0) {
331 *PageAttribute
= Page2M
;
332 return &L2PageTable
[Index2
];
336 L1PageTable
= (UINT64
*)(UINTN
)(L2PageTable
[Index2
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
337 if ((L1PageTable
[Index1
] == 0) && (Address
!= 0)) {
338 *PageAttribute
= PageNone
;
341 *PageAttribute
= Page4K
;
342 return &L1PageTable
[Index1
];
346 Return memory attributes of page entry.
348 @param[in] PageEntry The page entry.
350 @return Memory attributes of page entry.
353 GetAttributesFromPageEntry (
359 if ((*PageEntry
& IA32_PG_P
) == 0) {
360 Attributes
|= EFI_MEMORY_RP
;
362 if ((*PageEntry
& IA32_PG_RW
) == 0) {
363 Attributes
|= EFI_MEMORY_RO
;
365 if ((*PageEntry
& IA32_PG_NX
) != 0) {
366 Attributes
|= EFI_MEMORY_XP
;
372 Modify memory attributes of page entry.
374 @param[in] PagingContext The paging context.
375 @param[in] PageEntry The page entry.
376 @param[in] Attributes The bit mask of attributes to modify for the memory region.
377 @param[in] PageAction The page action.
378 @param[out] IsModified TRUE means page table modified. FALSE means page table not modified.
381 ConvertPageEntryAttribute (
382 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
,
383 IN UINT64
*PageEntry
,
384 IN UINT64 Attributes
,
385 IN PAGE_ACTION PageAction
,
386 OUT BOOLEAN
*IsModified
389 UINT64 CurrentPageEntry
;
392 CurrentPageEntry
= *PageEntry
;
393 NewPageEntry
= CurrentPageEntry
;
394 if ((Attributes
& EFI_MEMORY_RP
) != 0) {
395 switch (PageAction
) {
396 case PageActionAssign
:
398 NewPageEntry
&= ~(UINT64
)IA32_PG_P
;
400 case PageActionClear
:
401 NewPageEntry
|= IA32_PG_P
;
405 switch (PageAction
) {
406 case PageActionAssign
:
407 NewPageEntry
|= IA32_PG_P
;
410 case PageActionClear
:
414 if ((Attributes
& EFI_MEMORY_RO
) != 0) {
415 switch (PageAction
) {
416 case PageActionAssign
:
418 NewPageEntry
&= ~(UINT64
)IA32_PG_RW
;
420 case PageActionClear
:
421 NewPageEntry
|= IA32_PG_RW
;
425 switch (PageAction
) {
426 case PageActionAssign
:
427 NewPageEntry
|= IA32_PG_RW
;
430 case PageActionClear
:
434 if ((PagingContext
->ContextData
.Ia32
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED
) != 0) {
435 if ((Attributes
& EFI_MEMORY_XP
) != 0) {
436 switch (PageAction
) {
437 case PageActionAssign
:
439 NewPageEntry
|= IA32_PG_NX
;
441 case PageActionClear
:
442 NewPageEntry
&= ~IA32_PG_NX
;
446 switch (PageAction
) {
447 case PageActionAssign
:
448 NewPageEntry
&= ~IA32_PG_NX
;
451 case PageActionClear
:
456 *PageEntry
= NewPageEntry
;
457 if (CurrentPageEntry
!= NewPageEntry
) {
459 DEBUG ((DEBUG_VERBOSE
, "ConvertPageEntryAttribute 0x%lx", CurrentPageEntry
));
460 DEBUG ((DEBUG_VERBOSE
, "->0x%lx\n", NewPageEntry
));
467 This function returns if there is need to split page entry.
469 @param[in] BaseAddress The base address to be checked.
470 @param[in] Length The length to be checked.
471 @param[in] PageEntry The page entry to be checked.
472 @param[in] PageAttribute The page attribute of the page entry.
474 @retval SplitAttributes on if there is need to split page entry.
478 IN PHYSICAL_ADDRESS BaseAddress
,
480 IN UINT64
*PageEntry
,
481 IN PAGE_ATTRIBUTE PageAttribute
484 UINT64 PageEntryLength
;
486 PageEntryLength
= PageAttributeToLength (PageAttribute
);
488 if (((BaseAddress
& (PageEntryLength
- 1)) == 0) && (Length
>= PageEntryLength
)) {
492 if (((BaseAddress
& PAGING_2M_MASK
) != 0) || (Length
< SIZE_2MB
)) {
500 This function splits one page entry to small page entries.
502 @param[in] PageEntry The page entry to be splitted.
503 @param[in] PageAttribute The page attribute of the page entry.
504 @param[in] SplitAttribute How to split the page entry.
505 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
507 @retval RETURN_SUCCESS The page entry is splitted.
508 @retval RETURN_UNSUPPORTED The page entry does not support to be splitted.
509 @retval RETURN_OUT_OF_RESOURCES No resource to split page entry.
513 IN UINT64
*PageEntry
,
514 IN PAGE_ATTRIBUTE PageAttribute
,
515 IN PAGE_ATTRIBUTE SplitAttribute
,
516 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc
520 UINT64
*NewPageEntry
;
522 UINT64 AddressEncMask
;
524 ASSERT (PageAttribute
== Page2M
|| PageAttribute
== Page1G
);
526 ASSERT (AllocatePagesFunc
!= NULL
);
528 // Make sure AddressEncMask is contained to smallest supported address field.
530 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
532 if (PageAttribute
== Page2M
) {
536 ASSERT (SplitAttribute
== Page4K
);
537 if (SplitAttribute
== Page4K
) {
538 NewPageEntry
= AllocatePagesFunc (1);
539 DEBUG ((DEBUG_VERBOSE
, "Split - 0x%x\n", NewPageEntry
));
540 if (NewPageEntry
== NULL
) {
541 return RETURN_OUT_OF_RESOURCES
;
543 BaseAddress
= *PageEntry
& ~AddressEncMask
& PAGING_2M_ADDRESS_MASK_64
;
544 for (Index
= 0; Index
< SIZE_4KB
/ sizeof(UINT64
); Index
++) {
545 NewPageEntry
[Index
] = (BaseAddress
+ SIZE_4KB
* Index
) | AddressEncMask
| ((*PageEntry
) & PAGE_PROGATE_BITS
);
547 (*PageEntry
) = (UINT64
)(UINTN
)NewPageEntry
| AddressEncMask
| ((*PageEntry
) & PAGE_ATTRIBUTE_BITS
);
548 return RETURN_SUCCESS
;
550 return RETURN_UNSUPPORTED
;
552 } else if (PageAttribute
== Page1G
) {
555 // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
557 ASSERT (SplitAttribute
== Page2M
|| SplitAttribute
== Page4K
);
558 if ((SplitAttribute
== Page2M
|| SplitAttribute
== Page4K
)) {
559 NewPageEntry
= AllocatePagesFunc (1);
560 DEBUG ((DEBUG_VERBOSE
, "Split - 0x%x\n", NewPageEntry
));
561 if (NewPageEntry
== NULL
) {
562 return RETURN_OUT_OF_RESOURCES
;
564 BaseAddress
= *PageEntry
& ~AddressEncMask
& PAGING_1G_ADDRESS_MASK_64
;
565 for (Index
= 0; Index
< SIZE_4KB
/ sizeof(UINT64
); Index
++) {
566 NewPageEntry
[Index
] = (BaseAddress
+ SIZE_2MB
* Index
) | AddressEncMask
| IA32_PG_PS
| ((*PageEntry
) & PAGE_PROGATE_BITS
);
568 (*PageEntry
) = (UINT64
)(UINTN
)NewPageEntry
| AddressEncMask
| ((*PageEntry
) & PAGE_ATTRIBUTE_BITS
);
569 return RETURN_SUCCESS
;
571 return RETURN_UNSUPPORTED
;
574 return RETURN_UNSUPPORTED
;
579 Check the WP status in CR0 register. This bit is used to lock or unlock write
580 access to pages marked as read-only.
582 @retval TRUE Write protection is enabled.
583 @retval FALSE Write protection is disabled.
586 IsReadOnlyPageWriteProtected (
591 // To avoid unforseen consequences, don't touch paging settings in SMM mode
595 return ((AsmReadCr0 () & CR0_WP
) != 0);
601 Disable Write Protect on pages marked as read-only.
604 DisableReadOnlyPageWriteProtect (
609 // To avoid unforseen consequences, don't touch paging settings in SMM mode
613 AsmWriteCr0 (AsmReadCr0 () & ~CR0_WP
);
618 Enable Write Protect on pages marked as read-only.
621 EnableReadOnlyPageWriteProtect (
626 // To avoid unforseen consequences, don't touch paging settings in SMM mode
630 AsmWriteCr0 (AsmReadCr0 () | CR0_WP
);
635 This function modifies the page attributes for the memory region specified by BaseAddress and
636 Length from their current attributes to the attributes specified by Attributes.
638 Caller should make sure BaseAddress and Length is at page boundary.
640 @param[in] PagingContext The paging context. NULL means get page table from current CPU context.
641 @param[in] BaseAddress The physical address that is the start address of a memory region.
642 @param[in] Length The size in bytes of the memory region.
643 @param[in] Attributes The bit mask of attributes to modify for the memory region.
644 @param[in] PageAction The page action.
645 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
646 NULL mean page split is unsupported.
647 @param[out] IsSplitted TRUE means page table splitted. FALSE means page table not splitted.
648 @param[out] IsModified TRUE means page table modified. FALSE means page table not modified.
650 @retval RETURN_SUCCESS The attributes were modified for the memory region.
651 @retval RETURN_ACCESS_DENIED The attributes for the memory resource range specified by
652 BaseAddress and Length cannot be modified.
653 @retval RETURN_INVALID_PARAMETER Length is zero.
654 Attributes specified an illegal combination of attributes that
655 cannot be set together.
656 @retval RETURN_OUT_OF_RESOURCES There are not enough system resources to modify the attributes of
657 the memory resource range.
658 @retval RETURN_UNSUPPORTED The processor does not support one or more bytes of the memory
659 resource range specified by BaseAddress and Length.
660 The bit mask of attributes is not support for the memory resource
661 range specified by BaseAddress and Length.
664 ConvertMemoryPageAttributes (
665 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext OPTIONAL
,
666 IN PHYSICAL_ADDRESS BaseAddress
,
668 IN UINT64 Attributes
,
669 IN PAGE_ACTION PageAction
,
670 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc OPTIONAL
,
671 OUT BOOLEAN
*IsSplitted
, OPTIONAL
672 OUT BOOLEAN
*IsModified OPTIONAL
675 PAGE_TABLE_LIB_PAGING_CONTEXT CurrentPagingContext
;
677 PAGE_ATTRIBUTE PageAttribute
;
678 UINTN PageEntryLength
;
679 PAGE_ATTRIBUTE SplitAttribute
;
680 RETURN_STATUS Status
;
681 BOOLEAN IsEntryModified
;
684 if ((BaseAddress
& (SIZE_4KB
- 1)) != 0) {
685 DEBUG ((DEBUG_ERROR
, "BaseAddress(0x%lx) is not aligned!\n", BaseAddress
));
686 return EFI_UNSUPPORTED
;
688 if ((Length
& (SIZE_4KB
- 1)) != 0) {
689 DEBUG ((DEBUG_ERROR
, "Length(0x%lx) is not aligned!\n", Length
));
690 return EFI_UNSUPPORTED
;
693 DEBUG ((DEBUG_ERROR
, "Length is 0!\n"));
694 return RETURN_INVALID_PARAMETER
;
697 if ((Attributes
& ~(EFI_MEMORY_RP
| EFI_MEMORY_RO
| EFI_MEMORY_XP
)) != 0) {
698 DEBUG ((DEBUG_ERROR
, "Attributes(0x%lx) has unsupported bit\n", Attributes
));
699 return EFI_UNSUPPORTED
;
702 if (PagingContext
== NULL
) {
703 GetCurrentPagingContext (&CurrentPagingContext
);
705 CopyMem (&CurrentPagingContext
, PagingContext
, sizeof(CurrentPagingContext
));
707 switch(CurrentPagingContext
.MachineType
) {
708 case IMAGE_FILE_MACHINE_I386
:
709 if (CurrentPagingContext
.ContextData
.Ia32
.PageTableBase
== 0) {
710 if (Attributes
== 0) {
713 DEBUG ((DEBUG_ERROR
, "PageTable is 0!\n"));
714 return EFI_UNSUPPORTED
;
717 if ((CurrentPagingContext
.ContextData
.Ia32
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) == 0) {
718 DEBUG ((DEBUG_ERROR
, "Non-PAE Paging!\n"));
719 return EFI_UNSUPPORTED
;
721 if ((BaseAddress
+ Length
) > BASE_4GB
) {
722 DEBUG ((DEBUG_ERROR
, "Beyond 4GB memory in 32-bit mode!\n"));
723 return EFI_UNSUPPORTED
;
726 case IMAGE_FILE_MACHINE_X64
:
727 ASSERT (CurrentPagingContext
.ContextData
.X64
.PageTableBase
!= 0);
731 return EFI_UNSUPPORTED
;
735 // DEBUG ((DEBUG_ERROR, "ConvertMemoryPageAttributes(%x) - %016lx, %016lx, %02lx\n", IsSet, BaseAddress, Length, Attributes));
737 if (IsSplitted
!= NULL
) {
740 if (IsModified
!= NULL
) {
743 if (AllocatePagesFunc
== NULL
) {
744 AllocatePagesFunc
= AllocatePageTableMemory
;
748 // Make sure that the page table is changeable.
750 IsWpEnabled
= IsReadOnlyPageWriteProtected ();
752 DisableReadOnlyPageWriteProtect ();
756 // Below logic is to check 2M/4K page to make sure we donot waist memory.
758 Status
= EFI_SUCCESS
;
759 while (Length
!= 0) {
760 PageEntry
= GetPageTableEntry (&CurrentPagingContext
, BaseAddress
, &PageAttribute
);
761 if (PageEntry
== NULL
) {
762 Status
= RETURN_UNSUPPORTED
;
765 PageEntryLength
= PageAttributeToLength (PageAttribute
);
766 SplitAttribute
= NeedSplitPage (BaseAddress
, Length
, PageEntry
, PageAttribute
);
767 if (SplitAttribute
== PageNone
) {
768 ConvertPageEntryAttribute (&CurrentPagingContext
, PageEntry
, Attributes
, PageAction
, &IsEntryModified
);
769 if (IsEntryModified
) {
770 if (IsModified
!= NULL
) {
775 // Convert success, move to next
777 BaseAddress
+= PageEntryLength
;
778 Length
-= PageEntryLength
;
780 if (AllocatePagesFunc
== NULL
) {
781 Status
= RETURN_UNSUPPORTED
;
784 Status
= SplitPage (PageEntry
, PageAttribute
, SplitAttribute
, AllocatePagesFunc
);
785 if (RETURN_ERROR (Status
)) {
786 Status
= RETURN_UNSUPPORTED
;
789 if (IsSplitted
!= NULL
) {
792 if (IsModified
!= NULL
) {
796 // Just split current page
797 // Convert success in next around
804 // Restore page table write protection, if any.
807 EnableReadOnlyPageWriteProtect ();
813 This function assigns the page attributes for the memory region specified by BaseAddress and
814 Length from their current attributes to the attributes specified by Attributes.
816 Caller should make sure BaseAddress and Length is at page boundary.
818 Caller need guarentee the TPL <= TPL_NOTIFY, if there is split page request.
820 @param[in] PagingContext The paging context. NULL means get page table from current CPU context.
821 @param[in] BaseAddress The physical address that is the start address of a memory region.
822 @param[in] Length The size in bytes of the memory region.
823 @param[in] Attributes The bit mask of attributes to set for the memory region.
824 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
825 NULL mean page split is unsupported.
827 @retval RETURN_SUCCESS The attributes were cleared for the memory region.
828 @retval RETURN_ACCESS_DENIED The attributes for the memory resource range specified by
829 BaseAddress and Length cannot be modified.
830 @retval RETURN_INVALID_PARAMETER Length is zero.
831 Attributes specified an illegal combination of attributes that
832 cannot be set together.
833 @retval RETURN_OUT_OF_RESOURCES There are not enough system resources to modify the attributes of
834 the memory resource range.
835 @retval RETURN_UNSUPPORTED The processor does not support one or more bytes of the memory
836 resource range specified by BaseAddress and Length.
837 The bit mask of attributes is not support for the memory resource
838 range specified by BaseAddress and Length.
842 AssignMemoryPageAttributes (
843 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext OPTIONAL
,
844 IN PHYSICAL_ADDRESS BaseAddress
,
846 IN UINT64 Attributes
,
847 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc OPTIONAL
850 RETURN_STATUS Status
;
854 // DEBUG((DEBUG_INFO, "AssignMemoryPageAttributes: 0x%lx - 0x%lx (0x%lx)\n", BaseAddress, Length, Attributes));
855 Status
= ConvertMemoryPageAttributes (PagingContext
, BaseAddress
, Length
, Attributes
, PageActionAssign
, AllocatePagesFunc
, &IsSplitted
, &IsModified
);
856 if (!EFI_ERROR(Status
)) {
857 if ((PagingContext
== NULL
) && IsModified
) {
859 // Flush TLB as last step.
861 // Note: Since APs will always init CR3 register in HLT loop mode or do
862 // TLB flush in MWAIT loop mode, there's no need to flush TLB for them
873 Check if Execute Disable feature is enabled or not.
876 IsExecuteDisableEnabled (
880 MSR_CORE_IA32_EFER_REGISTER MsrEfer
;
882 MsrEfer
.Uint64
= AsmReadMsr64 (MSR_IA32_EFER
);
883 return (MsrEfer
.Bits
.NXE
== 1);
887 Update GCD memory space attributes according to current page table setup.
890 RefreshGcdMemoryAttributesFromPaging (
895 UINTN NumberOfDescriptors
;
896 EFI_GCD_MEMORY_SPACE_DESCRIPTOR
*MemorySpaceMap
;
897 PAGE_TABLE_LIB_PAGING_CONTEXT PagingContext
;
898 PAGE_ATTRIBUTE PageAttribute
;
901 UINT64 MemorySpaceLength
;
904 UINT64 PageStartAddress
;
907 UINT64 NewAttributes
;
911 // Assuming that memory space map returned is sorted already; otherwise sort
912 // them in the order of lowest address to highest address.
914 Status
= gDS
->GetMemorySpaceMap (&NumberOfDescriptors
, &MemorySpaceMap
);
915 ASSERT_EFI_ERROR (Status
);
917 GetCurrentPagingContext (&PagingContext
);
924 if (IsExecuteDisableEnabled ()) {
925 Capabilities
= EFI_MEMORY_RO
| EFI_MEMORY_RP
| EFI_MEMORY_XP
;
927 Capabilities
= EFI_MEMORY_RO
| EFI_MEMORY_RP
;
930 for (Index
= 0; Index
< NumberOfDescriptors
; Index
++) {
931 if (MemorySpaceMap
[Index
].GcdMemoryType
== EfiGcdMemoryTypeNonExistent
) {
936 // Sync the actual paging related capabilities back to GCD service first.
937 // As a side effect (good one), this can also help to avoid unnecessary
938 // memory map entries due to the different capabilities of the same type
939 // memory, such as multiple RT_CODE and RT_DATA entries in memory map,
940 // which could cause boot failure of some old Linux distro (before v4.3).
942 Status
= gDS
->SetMemorySpaceCapabilities (
943 MemorySpaceMap
[Index
].BaseAddress
,
944 MemorySpaceMap
[Index
].Length
,
945 MemorySpaceMap
[Index
].Capabilities
| Capabilities
947 if (EFI_ERROR (Status
)) {
949 // If we cannot udpate the capabilities, we cannot update its
950 // attributes either. So just simply skip current block of memory.
954 "Failed to update capability: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
955 (UINT64
)Index
, MemorySpaceMap
[Index
].BaseAddress
,
956 MemorySpaceMap
[Index
].BaseAddress
+ MemorySpaceMap
[Index
].Length
- 1,
957 MemorySpaceMap
[Index
].Capabilities
,
958 MemorySpaceMap
[Index
].Capabilities
| Capabilities
963 if (MemorySpaceMap
[Index
].BaseAddress
>= (BaseAddress
+ PageLength
)) {
965 // Current memory space starts at a new page. Resetting PageLength will
966 // trigger a retrieval of page attributes at new address.
971 // In case current memory space is not adjacent to last one
973 PageLength
-= (MemorySpaceMap
[Index
].BaseAddress
- BaseAddress
);
977 // Sync actual page attributes to GCD
979 BaseAddress
= MemorySpaceMap
[Index
].BaseAddress
;
980 MemorySpaceLength
= MemorySpaceMap
[Index
].Length
;
981 while (MemorySpaceLength
> 0) {
982 if (PageLength
== 0) {
983 PageEntry
= GetPageTableEntry (&PagingContext
, BaseAddress
, &PageAttribute
);
984 if (PageEntry
== NULL
) {
989 // Note current memory space might start in the middle of a page
991 PageStartAddress
= (*PageEntry
) & (UINT64
)PageAttributeToMask(PageAttribute
);
992 PageLength
= PageAttributeToLength (PageAttribute
) - (BaseAddress
- PageStartAddress
);
993 Attributes
= GetAttributesFromPageEntry (PageEntry
);
996 Length
= MIN (PageLength
, MemorySpaceLength
);
997 if (Attributes
!= (MemorySpaceMap
[Index
].Attributes
&
998 EFI_MEMORY_PAGETYPE_MASK
)) {
999 NewAttributes
= (MemorySpaceMap
[Index
].Attributes
&
1000 ~EFI_MEMORY_PAGETYPE_MASK
) | Attributes
;
1001 Status
= gDS
->SetMemorySpaceAttributes (
1006 ASSERT_EFI_ERROR (Status
);
1009 "Updated memory space attribute: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
1010 (UINT64
)Index
, BaseAddress
, BaseAddress
+ Length
- 1,
1011 MemorySpaceMap
[Index
].Attributes
,
1016 PageLength
-= Length
;
1017 MemorySpaceLength
-= Length
;
1018 BaseAddress
+= Length
;
1022 FreePool (MemorySpaceMap
);
1026 Initialize a buffer pool for page table use only.
1028 To reduce the potential split operation on page table, the pages reserved for
1029 page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
1030 at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
1031 initialized with number of pages greater than or equal to the given PoolPages.
1033 Once the pages in the pool are used up, this method should be called again to
1034 reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. Usually this won't happen
1037 @param[in] PoolPages The least page number of the pool to be created.
1039 @retval TRUE The pool is initialized successfully.
1040 @retval FALSE The memory is out of resource.
1043 InitializePageTablePool (
1051 // Do not allow re-entrance.
1053 if (mPageTablePoolLock
) {
1057 mPageTablePoolLock
= TRUE
;
1061 // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
1064 PoolPages
+= 1; // Add one page for header.
1065 PoolPages
= ((PoolPages
- 1) / PAGE_TABLE_POOL_UNIT_PAGES
+ 1) *
1066 PAGE_TABLE_POOL_UNIT_PAGES
;
1067 Buffer
= AllocateAlignedPages (PoolPages
, PAGE_TABLE_POOL_ALIGNMENT
);
1068 if (Buffer
== NULL
) {
1069 DEBUG ((DEBUG_ERROR
, "ERROR: Out of aligned pages\r\n"));
1075 "Paging: added %lu pages to page table pool\r\n",
1080 // Link all pools into a list for easier track later.
1082 if (mPageTablePool
== NULL
) {
1083 mPageTablePool
= Buffer
;
1084 mPageTablePool
->NextPool
= mPageTablePool
;
1086 ((PAGE_TABLE_POOL
*)Buffer
)->NextPool
= mPageTablePool
->NextPool
;
1087 mPageTablePool
->NextPool
= Buffer
;
1088 mPageTablePool
= Buffer
;
1092 // Reserve one page for pool header.
1094 mPageTablePool
->FreePages
= PoolPages
- 1;
1095 mPageTablePool
->Offset
= EFI_PAGES_TO_SIZE (1);
1098 // Mark the whole pool pages as read-only.
1100 ConvertMemoryPageAttributes (
1102 (PHYSICAL_ADDRESS
)(UINTN
)Buffer
,
1103 EFI_PAGES_TO_SIZE (PoolPages
),
1106 AllocatePageTableMemory
,
1110 ASSERT (IsModified
== TRUE
);
1113 mPageTablePoolLock
= FALSE
;
1118 This API provides a way to allocate memory for page table.
1120 This API can be called more than once to allocate memory for page tables.
1122 Allocates the number of 4KB pages and returns a pointer to the allocated
1123 buffer. The buffer returned is aligned on a 4KB boundary.
1125 If Pages is 0, then NULL is returned.
1126 If there is not enough memory remaining to satisfy the request, then NULL is
1129 @param Pages The number of 4 KB pages to allocate.
1131 @return A pointer to the allocated buffer or NULL if allocation fails.
1136 AllocatePageTableMemory (
1147 // Renew the pool if necessary.
1149 if (mPageTablePool
== NULL
||
1150 Pages
> mPageTablePool
->FreePages
) {
1151 if (!InitializePageTablePool (Pages
)) {
1156 Buffer
= (UINT8
*)mPageTablePool
+ mPageTablePool
->Offset
;
1158 mPageTablePool
->Offset
+= EFI_PAGES_TO_SIZE (Pages
);
1159 mPageTablePool
->FreePages
-= Pages
;
1165 Special handler for #DB exception, which will restore the page attributes
1166 (not-present). It should work with #PF handler which will set pages to
1169 @param ExceptionType Exception type.
1170 @param SystemContext Pointer to EFI_SYSTEM_CONTEXT.
1175 DebugExceptionHandler (
1176 IN EFI_EXCEPTION_TYPE ExceptionType
,
1177 IN EFI_SYSTEM_CONTEXT SystemContext
1182 BOOLEAN IsWpEnabled
;
1184 MpInitLibWhoAmI (&CpuIndex
);
1187 // Clear last PF entries
1189 IsWpEnabled
= IsReadOnlyPageWriteProtected ();
1191 DisableReadOnlyPageWriteProtect ();
1194 for (PFEntry
= 0; PFEntry
< mPFEntryCount
[CpuIndex
]; PFEntry
++) {
1195 if (mLastPFEntryPointer
[CpuIndex
][PFEntry
] != NULL
) {
1196 *mLastPFEntryPointer
[CpuIndex
][PFEntry
] &= ~(UINT64
)IA32_PG_P
;
1201 EnableReadOnlyPageWriteProtect ();
1205 // Reset page fault exception count for next page fault.
1207 mPFEntryCount
[CpuIndex
] = 0;
1215 // Clear TF in EFLAGS
1217 if (mPagingContext
.MachineType
== IMAGE_FILE_MACHINE_I386
) {
1218 SystemContext
.SystemContextIa32
->Eflags
&= (UINT32
)~BIT8
;
1220 SystemContext
.SystemContextX64
->Rflags
&= (UINT64
)~BIT8
;
1225 Special handler for #PF exception, which will set the pages which caused
1226 #PF to be 'present'. The attribute of those pages should be restored in
1227 the subsequent #DB handler.
1229 @param ExceptionType Exception type.
1230 @param SystemContext Pointer to EFI_SYSTEM_CONTEXT.
1235 PageFaultExceptionHandler (
1236 IN EFI_EXCEPTION_TYPE ExceptionType
,
1237 IN EFI_SYSTEM_CONTEXT SystemContext
1242 PAGE_TABLE_LIB_PAGING_CONTEXT PagingContext
;
1243 PAGE_ATTRIBUTE PageAttribute
;
1249 BOOLEAN NonStopMode
;
1251 PFAddress
= AsmReadCr2 () & ~EFI_PAGE_MASK
;
1252 if (PFAddress
< BASE_4KB
) {
1253 NonStopMode
= NULL_DETECTION_NONSTOP_MODE
? TRUE
: FALSE
;
1255 NonStopMode
= HEAP_GUARD_NONSTOP_MODE
? TRUE
: FALSE
;
1259 MpInitLibWhoAmI (&CpuIndex
);
1260 GetCurrentPagingContext (&PagingContext
);
1262 // Memory operation cross page boundary, like "rep mov" instruction, will
1263 // cause infinite loop between this and Debug Trap handler. We have to make
1264 // sure that current page and the page followed are both in PRESENT state.
1267 while (PageNumber
> 0) {
1268 PageEntry
= GetPageTableEntry (&PagingContext
, PFAddress
, &PageAttribute
);
1269 ASSERT(PageEntry
!= NULL
);
1271 if (PageEntry
!= NULL
) {
1272 Attributes
= GetAttributesFromPageEntry (PageEntry
);
1273 if ((Attributes
& EFI_MEMORY_RP
) != 0) {
1274 Attributes
&= ~EFI_MEMORY_RP
;
1275 Status
= AssignMemoryPageAttributes (&PagingContext
, PFAddress
,
1276 EFI_PAGE_SIZE
, Attributes
, NULL
);
1277 if (!EFI_ERROR(Status
)) {
1278 Index
= mPFEntryCount
[CpuIndex
];
1280 // Re-retrieve page entry because above calling might update page
1281 // table due to table split.
1283 PageEntry
= GetPageTableEntry (&PagingContext
, PFAddress
, &PageAttribute
);
1284 mLastPFEntryPointer
[CpuIndex
][Index
++] = PageEntry
;
1285 mPFEntryCount
[CpuIndex
] = Index
;
1290 PFAddress
+= EFI_PAGE_SIZE
;
1296 // Initialize the serial port before dumping.
1298 SerialPortInitialize ();
1300 // Display ExceptionType, CPU information and Image information
1302 DumpCpuContext (ExceptionType
, SystemContext
);
1309 Initialize the Page Table lib.
1312 InitializePageTableLib (
1316 PAGE_TABLE_LIB_PAGING_CONTEXT CurrentPagingContext
;
1318 GetCurrentPagingContext (&CurrentPagingContext
);
1321 // Reserve memory of page tables for future uses, if paging is enabled.
1323 if (CurrentPagingContext
.ContextData
.X64
.PageTableBase
!= 0 &&
1324 (CurrentPagingContext
.ContextData
.Ia32
.Attributes
&
1325 PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) != 0) {
1326 DisableReadOnlyPageWriteProtect ();
1327 InitializePageTablePool (1);
1328 EnableReadOnlyPageWriteProtect ();
1331 if (HEAP_GUARD_NONSTOP_MODE
|| NULL_DETECTION_NONSTOP_MODE
) {
1332 mPFEntryCount
= (UINTN
*)AllocateZeroPool (sizeof (UINTN
) * mNumberOfProcessors
);
1333 ASSERT (mPFEntryCount
!= NULL
);
1335 mLastPFEntryPointer
= (UINT64
*(*)[MAX_PF_ENTRY_COUNT
])
1336 AllocateZeroPool (sizeof (mLastPFEntryPointer
[0]) * mNumberOfProcessors
);
1337 ASSERT (mLastPFEntryPointer
!= NULL
);
1340 DEBUG ((DEBUG_INFO
, "CurrentPagingContext:\n", CurrentPagingContext
.MachineType
));
1341 DEBUG ((DEBUG_INFO
, " MachineType - 0x%x\n", CurrentPagingContext
.MachineType
));
1342 DEBUG ((DEBUG_INFO
, " PageTableBase - 0x%x\n", CurrentPagingContext
.ContextData
.X64
.PageTableBase
));
1343 DEBUG ((DEBUG_INFO
, " Attributes - 0x%x\n", CurrentPagingContext
.ContextData
.X64
.Attributes
));