2 Page table management support.
4 Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
7 This program and the accompanying materials
8 are licensed and made available under the terms and conditions of the BSD License
9 which accompanies this distribution. The full text of the license may be found at
10 http://opensource.org/licenses/bsd-license.php
12 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
13 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
19 #include <Library/BaseLib.h>
20 #include <Library/CpuLib.h>
21 #include <Library/BaseMemoryLib.h>
22 #include <Library/MemoryAllocationLib.h>
23 #include <Library/DebugLib.h>
24 #include <Library/UefiBootServicesTableLib.h>
25 #include <Protocol/MpService.h>
26 #include <Protocol/SmmBase2.h>
27 #include <Register/Cpuid.h>
28 #include <Register/Msr.h>
31 #include "CpuPageTable.h"
44 #define IA32_PG_P BIT0
45 #define IA32_PG_RW BIT1
46 #define IA32_PG_U BIT2
47 #define IA32_PG_WT BIT3
48 #define IA32_PG_CD BIT4
49 #define IA32_PG_A BIT5
50 #define IA32_PG_D BIT6
51 #define IA32_PG_PS BIT7
52 #define IA32_PG_PAT_2M BIT12
53 #define IA32_PG_PAT_4K IA32_PG_PS
54 #define IA32_PG_PMNT BIT62
55 #define IA32_PG_NX BIT63
57 #define PAGE_ATTRIBUTE_BITS (IA32_PG_D | IA32_PG_A | IA32_PG_U | IA32_PG_RW | IA32_PG_P)
59 // Bits 1, 2, 5, 6 are reserved in the IA32 PAE PDPTE
60 // X64 PAE PDPTE does not have such restriction
62 #define IA32_PAE_PDPTE_ATTRIBUTE_BITS (IA32_PG_P)
64 #define PAGE_PROGATE_BITS (IA32_PG_NX | PAGE_ATTRIBUTE_BITS)
66 #define PAGING_4K_MASK 0xFFF
67 #define PAGING_2M_MASK 0x1FFFFF
68 #define PAGING_1G_MASK 0x3FFFFFFF
70 #define PAGING_PAE_INDEX_MASK 0x1FF
72 #define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull
73 #define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull
74 #define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull
84 PAGE_ATTRIBUTE Attribute
;
87 } PAGE_ATTRIBUTE_TABLE
;
95 PAGE_ATTRIBUTE_TABLE mPageAttributeTable
[] = {
96 {Page4K
, SIZE_4KB
, PAGING_4K_ADDRESS_MASK_64
},
97 {Page2M
, SIZE_2MB
, PAGING_2M_ADDRESS_MASK_64
},
98 {Page1G
, SIZE_1GB
, PAGING_1G_ADDRESS_MASK_64
},
101 PAGE_TABLE_POOL
*mPageTablePool
= NULL
;
102 PAGE_TABLE_LIB_PAGING_CONTEXT mPagingContext
;
103 EFI_SMM_BASE2_PROTOCOL
*mSmmBase2
= NULL
;
106 Check if current execution environment is in SMM mode or not, via
107 EFI_SMM_BASE2_PROTOCOL.
109 This is necessary because of the fact that MdePkg\Library\SmmMemoryAllocationLib
110 supports to free memory outside SMRAM. The library will call gBS->FreePool() or
111 gBS->FreePages() and then SetMemorySpaceAttributes interface in turn to change
112 memory paging attributes during free operation, if some memory related features
113 are enabled (like Heap Guard).
115 This means that SetMemorySpaceAttributes() has chance to run in SMM mode. This
116 will cause incorrect result because SMM mode always loads its own page tables,
117 which are usually different from DXE. This function can be used to detect such
118 situation and help to avoid further misoperations.
120 @retval TRUE In SMM mode.
121 @retval FALSE Not in SMM mode.
131 if (mSmmBase2
== NULL
) {
132 gBS
->LocateProtocol (&gEfiSmmBase2ProtocolGuid
, NULL
, (VOID
**)&mSmmBase2
);
135 if (mSmmBase2
!= NULL
) {
136 mSmmBase2
->InSmm (mSmmBase2
, &InSmm
);
143 Return current paging context.
145 @param[in,out] PagingContext The paging context.
148 GetCurrentPagingContext (
149 IN OUT PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
153 CPUID_EXTENDED_CPU_SIG_EDX RegEdx
;
154 MSR_IA32_EFER_REGISTER MsrEfer
;
157 // Don't retrieve current paging context from processor if in SMM mode.
160 ZeroMem (&mPagingContext
, sizeof(mPagingContext
));
161 if (sizeof(UINTN
) == sizeof(UINT64
)) {
162 mPagingContext
.MachineType
= IMAGE_FILE_MACHINE_X64
;
164 mPagingContext
.MachineType
= IMAGE_FILE_MACHINE_I386
;
166 if ((AsmReadCr0 () & CR0_PG
) != 0) {
167 mPagingContext
.ContextData
.X64
.PageTableBase
= (AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64
);
169 mPagingContext
.ContextData
.X64
.PageTableBase
= 0;
172 if ((AsmReadCr4 () & CR4_PSE
) != 0) {
173 mPagingContext
.ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PSE
;
175 if ((AsmReadCr4 () & CR4_PAE
) != 0) {
176 mPagingContext
.ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
;
178 if ((AsmReadCr0 () & CR0_WP
) != 0) {
179 mPagingContext
.ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_WP_ENABLE
;
182 AsmCpuid (CPUID_EXTENDED_FUNCTION
, &RegEax
, NULL
, NULL
, NULL
);
183 if (RegEax
>= CPUID_EXTENDED_CPU_SIG
) {
184 AsmCpuid (CPUID_EXTENDED_CPU_SIG
, NULL
, NULL
, NULL
, &RegEdx
.Uint32
);
186 if (RegEdx
.Bits
.NX
!= 0) {
188 MsrEfer
.Uint64
= AsmReadMsr64(MSR_CORE_IA32_EFER
);
189 if (MsrEfer
.Bits
.NXE
!= 0) {
191 mPagingContext
.ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED
;
195 if (RegEdx
.Bits
.Page1GB
!= 0) {
196 mPagingContext
.ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAGE_1G_SUPPORT
;
202 // This can avoid getting SMM paging context if in SMM mode. We cannot assume
203 // SMM mode shares the same paging context as DXE.
205 CopyMem (PagingContext
, &mPagingContext
, sizeof (mPagingContext
));
209 Return length according to page attributes.
211 @param[in] PageAttributes The page attribute of the page entry.
213 @return The length of page entry.
216 PageAttributeToLength (
217 IN PAGE_ATTRIBUTE PageAttribute
221 for (Index
= 0; Index
< sizeof(mPageAttributeTable
)/sizeof(mPageAttributeTable
[0]); Index
++) {
222 if (PageAttribute
== mPageAttributeTable
[Index
].Attribute
) {
223 return (UINTN
)mPageAttributeTable
[Index
].Length
;
230 Return address mask according to page attributes.
232 @param[in] PageAttributes The page attribute of the page entry.
234 @return The address mask of page entry.
237 PageAttributeToMask (
238 IN PAGE_ATTRIBUTE PageAttribute
242 for (Index
= 0; Index
< sizeof(mPageAttributeTable
)/sizeof(mPageAttributeTable
[0]); Index
++) {
243 if (PageAttribute
== mPageAttributeTable
[Index
].Attribute
) {
244 return (UINTN
)mPageAttributeTable
[Index
].AddressMask
;
251 Return page table entry to match the address.
253 @param[in] PagingContext The paging context.
254 @param[in] Address The address to be checked.
255 @param[out] PageAttributes The page attribute of the page entry.
257 @return The page entry.
261 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
,
262 IN PHYSICAL_ADDRESS Address
,
263 OUT PAGE_ATTRIBUTE
*PageAttribute
274 UINT64 AddressEncMask
;
276 ASSERT (PagingContext
!= NULL
);
278 Index4
= ((UINTN
)RShiftU64 (Address
, 39)) & PAGING_PAE_INDEX_MASK
;
279 Index3
= ((UINTN
)Address
>> 30) & PAGING_PAE_INDEX_MASK
;
280 Index2
= ((UINTN
)Address
>> 21) & PAGING_PAE_INDEX_MASK
;
281 Index1
= ((UINTN
)Address
>> 12) & PAGING_PAE_INDEX_MASK
;
283 // Make sure AddressEncMask is contained to smallest supported address field.
285 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
287 if (PagingContext
->MachineType
== IMAGE_FILE_MACHINE_X64
) {
288 L4PageTable
= (UINT64
*)(UINTN
)PagingContext
->ContextData
.X64
.PageTableBase
;
289 if (L4PageTable
[Index4
] == 0) {
290 *PageAttribute
= PageNone
;
294 L3PageTable
= (UINT64
*)(UINTN
)(L4PageTable
[Index4
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
296 ASSERT((PagingContext
->ContextData
.Ia32
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) != 0);
297 L3PageTable
= (UINT64
*)(UINTN
)PagingContext
->ContextData
.Ia32
.PageTableBase
;
299 if (L3PageTable
[Index3
] == 0) {
300 *PageAttribute
= PageNone
;
303 if ((L3PageTable
[Index3
] & IA32_PG_PS
) != 0) {
305 *PageAttribute
= Page1G
;
306 return &L3PageTable
[Index3
];
309 L2PageTable
= (UINT64
*)(UINTN
)(L3PageTable
[Index3
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
310 if (L2PageTable
[Index2
] == 0) {
311 *PageAttribute
= PageNone
;
314 if ((L2PageTable
[Index2
] & IA32_PG_PS
) != 0) {
316 *PageAttribute
= Page2M
;
317 return &L2PageTable
[Index2
];
321 L1PageTable
= (UINT64
*)(UINTN
)(L2PageTable
[Index2
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
322 if ((L1PageTable
[Index1
] == 0) && (Address
!= 0)) {
323 *PageAttribute
= PageNone
;
326 *PageAttribute
= Page4K
;
327 return &L1PageTable
[Index1
];
331 Return memory attributes of page entry.
333 @param[in] PageEntry The page entry.
335 @return Memory attributes of page entry.
338 GetAttributesFromPageEntry (
344 if ((*PageEntry
& IA32_PG_P
) == 0) {
345 Attributes
|= EFI_MEMORY_RP
;
347 if ((*PageEntry
& IA32_PG_RW
) == 0) {
348 Attributes
|= EFI_MEMORY_RO
;
350 if ((*PageEntry
& IA32_PG_NX
) != 0) {
351 Attributes
|= EFI_MEMORY_XP
;
357 Modify memory attributes of page entry.
359 @param[in] PagingContext The paging context.
360 @param[in] PageEntry The page entry.
361 @param[in] Attributes The bit mask of attributes to modify for the memory region.
362 @param[in] PageAction The page action.
363 @param[out] IsModified TRUE means page table modified. FALSE means page table not modified.
366 ConvertPageEntryAttribute (
367 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
,
368 IN UINT64
*PageEntry
,
369 IN UINT64 Attributes
,
370 IN PAGE_ACTION PageAction
,
371 OUT BOOLEAN
*IsModified
374 UINT64 CurrentPageEntry
;
377 CurrentPageEntry
= *PageEntry
;
378 NewPageEntry
= CurrentPageEntry
;
379 if ((Attributes
& EFI_MEMORY_RP
) != 0) {
380 switch (PageAction
) {
381 case PageActionAssign
:
383 NewPageEntry
&= ~(UINT64
)IA32_PG_P
;
385 case PageActionClear
:
386 NewPageEntry
|= IA32_PG_P
;
390 switch (PageAction
) {
391 case PageActionAssign
:
392 NewPageEntry
|= IA32_PG_P
;
395 case PageActionClear
:
399 if ((Attributes
& EFI_MEMORY_RO
) != 0) {
400 switch (PageAction
) {
401 case PageActionAssign
:
403 NewPageEntry
&= ~(UINT64
)IA32_PG_RW
;
405 case PageActionClear
:
406 NewPageEntry
|= IA32_PG_RW
;
410 switch (PageAction
) {
411 case PageActionAssign
:
412 NewPageEntry
|= IA32_PG_RW
;
415 case PageActionClear
:
419 if ((PagingContext
->ContextData
.Ia32
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED
) != 0) {
420 if ((Attributes
& EFI_MEMORY_XP
) != 0) {
421 switch (PageAction
) {
422 case PageActionAssign
:
424 NewPageEntry
|= IA32_PG_NX
;
426 case PageActionClear
:
427 NewPageEntry
&= ~IA32_PG_NX
;
431 switch (PageAction
) {
432 case PageActionAssign
:
433 NewPageEntry
&= ~IA32_PG_NX
;
436 case PageActionClear
:
441 *PageEntry
= NewPageEntry
;
442 if (CurrentPageEntry
!= NewPageEntry
) {
444 DEBUG ((DEBUG_VERBOSE
, "ConvertPageEntryAttribute 0x%lx", CurrentPageEntry
));
445 DEBUG ((DEBUG_VERBOSE
, "->0x%lx\n", NewPageEntry
));
452 This function returns if there is need to split page entry.
454 @param[in] BaseAddress The base address to be checked.
455 @param[in] Length The length to be checked.
456 @param[in] PageEntry The page entry to be checked.
457 @param[in] PageAttribute The page attribute of the page entry.
459 @retval SplitAttributes on if there is need to split page entry.
463 IN PHYSICAL_ADDRESS BaseAddress
,
465 IN UINT64
*PageEntry
,
466 IN PAGE_ATTRIBUTE PageAttribute
469 UINT64 PageEntryLength
;
471 PageEntryLength
= PageAttributeToLength (PageAttribute
);
473 if (((BaseAddress
& (PageEntryLength
- 1)) == 0) && (Length
>= PageEntryLength
)) {
477 if (((BaseAddress
& PAGING_2M_MASK
) != 0) || (Length
< SIZE_2MB
)) {
485 This function splits one page entry to small page entries.
487 @param[in] PageEntry The page entry to be splitted.
488 @param[in] PageAttribute The page attribute of the page entry.
489 @param[in] SplitAttribute How to split the page entry.
490 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
492 @retval RETURN_SUCCESS The page entry is splitted.
493 @retval RETURN_UNSUPPORTED The page entry does not support to be splitted.
494 @retval RETURN_OUT_OF_RESOURCES No resource to split page entry.
498 IN UINT64
*PageEntry
,
499 IN PAGE_ATTRIBUTE PageAttribute
,
500 IN PAGE_ATTRIBUTE SplitAttribute
,
501 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc
505 UINT64
*NewPageEntry
;
507 UINT64 AddressEncMask
;
509 ASSERT (PageAttribute
== Page2M
|| PageAttribute
== Page1G
);
511 ASSERT (AllocatePagesFunc
!= NULL
);
513 // Make sure AddressEncMask is contained to smallest supported address field.
515 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
517 if (PageAttribute
== Page2M
) {
521 ASSERT (SplitAttribute
== Page4K
);
522 if (SplitAttribute
== Page4K
) {
523 NewPageEntry
= AllocatePagesFunc (1);
524 DEBUG ((DEBUG_INFO
, "Split - 0x%x\n", NewPageEntry
));
525 if (NewPageEntry
== NULL
) {
526 return RETURN_OUT_OF_RESOURCES
;
528 BaseAddress
= *PageEntry
& ~AddressEncMask
& PAGING_2M_ADDRESS_MASK_64
;
529 for (Index
= 0; Index
< SIZE_4KB
/ sizeof(UINT64
); Index
++) {
530 NewPageEntry
[Index
] = (BaseAddress
+ SIZE_4KB
* Index
) | AddressEncMask
| ((*PageEntry
) & PAGE_PROGATE_BITS
);
532 (*PageEntry
) = (UINT64
)(UINTN
)NewPageEntry
| AddressEncMask
| ((*PageEntry
) & PAGE_ATTRIBUTE_BITS
);
533 return RETURN_SUCCESS
;
535 return RETURN_UNSUPPORTED
;
537 } else if (PageAttribute
== Page1G
) {
540 // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
542 ASSERT (SplitAttribute
== Page2M
|| SplitAttribute
== Page4K
);
543 if ((SplitAttribute
== Page2M
|| SplitAttribute
== Page4K
)) {
544 NewPageEntry
= AllocatePagesFunc (1);
545 DEBUG ((DEBUG_INFO
, "Split - 0x%x\n", NewPageEntry
));
546 if (NewPageEntry
== NULL
) {
547 return RETURN_OUT_OF_RESOURCES
;
549 BaseAddress
= *PageEntry
& ~AddressEncMask
& PAGING_1G_ADDRESS_MASK_64
;
550 for (Index
= 0; Index
< SIZE_4KB
/ sizeof(UINT64
); Index
++) {
551 NewPageEntry
[Index
] = (BaseAddress
+ SIZE_2MB
* Index
) | AddressEncMask
| IA32_PG_PS
| ((*PageEntry
) & PAGE_PROGATE_BITS
);
553 (*PageEntry
) = (UINT64
)(UINTN
)NewPageEntry
| AddressEncMask
| ((*PageEntry
) & PAGE_ATTRIBUTE_BITS
);
554 return RETURN_SUCCESS
;
556 return RETURN_UNSUPPORTED
;
559 return RETURN_UNSUPPORTED
;
564 Check the WP status in CR0 register. This bit is used to lock or unlock write
565 access to pages marked as read-only.
567 @retval TRUE Write protection is enabled.
568 @retval FALSE Write protection is disabled.
571 IsReadOnlyPageWriteProtected (
576 // To avoid unforseen consequences, don't touch paging settings in SMM mode
580 return ((AsmReadCr0 () & CR0_WP
) != 0);
586 Disable Write Protect on pages marked as read-only.
589 DisableReadOnlyPageWriteProtect (
594 // To avoid unforseen consequences, don't touch paging settings in SMM mode
598 AsmWriteCr0 (AsmReadCr0 () & ~CR0_WP
);
603 Enable Write Protect on pages marked as read-only.
606 EnableReadOnlyPageWriteProtect (
611 // To avoid unforseen consequences, don't touch paging settings in SMM mode
615 AsmWriteCr0 (AsmReadCr0 () | CR0_WP
);
620 This function modifies the page attributes for the memory region specified by BaseAddress and
621 Length from their current attributes to the attributes specified by Attributes.
623 Caller should make sure BaseAddress and Length is at page boundary.
625 @param[in] PagingContext The paging context. NULL means get page table from current CPU context.
626 @param[in] BaseAddress The physical address that is the start address of a memory region.
627 @param[in] Length The size in bytes of the memory region.
628 @param[in] Attributes The bit mask of attributes to modify for the memory region.
629 @param[in] PageAction The page action.
630 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
631 NULL mean page split is unsupported.
632 @param[out] IsSplitted TRUE means page table splitted. FALSE means page table not splitted.
633 @param[out] IsModified TRUE means page table modified. FALSE means page table not modified.
635 @retval RETURN_SUCCESS The attributes were modified for the memory region.
636 @retval RETURN_ACCESS_DENIED The attributes for the memory resource range specified by
637 BaseAddress and Length cannot be modified.
638 @retval RETURN_INVALID_PARAMETER Length is zero.
639 Attributes specified an illegal combination of attributes that
640 cannot be set together.
641 @retval RETURN_OUT_OF_RESOURCES There are not enough system resources to modify the attributes of
642 the memory resource range.
643 @retval RETURN_UNSUPPORTED The processor does not support one or more bytes of the memory
644 resource range specified by BaseAddress and Length.
645 The bit mask of attributes is not support for the memory resource
646 range specified by BaseAddress and Length.
649 ConvertMemoryPageAttributes (
650 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext OPTIONAL
,
651 IN PHYSICAL_ADDRESS BaseAddress
,
653 IN UINT64 Attributes
,
654 IN PAGE_ACTION PageAction
,
655 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc OPTIONAL
,
656 OUT BOOLEAN
*IsSplitted
, OPTIONAL
657 OUT BOOLEAN
*IsModified OPTIONAL
660 PAGE_TABLE_LIB_PAGING_CONTEXT CurrentPagingContext
;
662 PAGE_ATTRIBUTE PageAttribute
;
663 UINTN PageEntryLength
;
664 PAGE_ATTRIBUTE SplitAttribute
;
665 RETURN_STATUS Status
;
666 BOOLEAN IsEntryModified
;
669 if ((BaseAddress
& (SIZE_4KB
- 1)) != 0) {
670 DEBUG ((DEBUG_ERROR
, "BaseAddress(0x%lx) is not aligned!\n", BaseAddress
));
671 return EFI_UNSUPPORTED
;
673 if ((Length
& (SIZE_4KB
- 1)) != 0) {
674 DEBUG ((DEBUG_ERROR
, "Length(0x%lx) is not aligned!\n", Length
));
675 return EFI_UNSUPPORTED
;
678 DEBUG ((DEBUG_ERROR
, "Length is 0!\n"));
679 return RETURN_INVALID_PARAMETER
;
682 if ((Attributes
& ~(EFI_MEMORY_RP
| EFI_MEMORY_RO
| EFI_MEMORY_XP
)) != 0) {
683 DEBUG ((DEBUG_ERROR
, "Attributes(0x%lx) has unsupported bit\n", Attributes
));
684 return EFI_UNSUPPORTED
;
687 if (PagingContext
== NULL
) {
688 GetCurrentPagingContext (&CurrentPagingContext
);
690 CopyMem (&CurrentPagingContext
, PagingContext
, sizeof(CurrentPagingContext
));
692 switch(CurrentPagingContext
.MachineType
) {
693 case IMAGE_FILE_MACHINE_I386
:
694 if (CurrentPagingContext
.ContextData
.Ia32
.PageTableBase
== 0) {
695 if (Attributes
== 0) {
698 DEBUG ((DEBUG_ERROR
, "PageTable is 0!\n"));
699 return EFI_UNSUPPORTED
;
702 if ((CurrentPagingContext
.ContextData
.Ia32
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) == 0) {
703 DEBUG ((DEBUG_ERROR
, "Non-PAE Paging!\n"));
704 return EFI_UNSUPPORTED
;
706 if ((BaseAddress
+ Length
) > BASE_4GB
) {
707 DEBUG ((DEBUG_ERROR
, "Beyond 4GB memory in 32-bit mode!\n"));
708 return EFI_UNSUPPORTED
;
711 case IMAGE_FILE_MACHINE_X64
:
712 ASSERT (CurrentPagingContext
.ContextData
.X64
.PageTableBase
!= 0);
716 return EFI_UNSUPPORTED
;
720 // DEBUG ((DEBUG_ERROR, "ConvertMemoryPageAttributes(%x) - %016lx, %016lx, %02lx\n", IsSet, BaseAddress, Length, Attributes));
722 if (IsSplitted
!= NULL
) {
725 if (IsModified
!= NULL
) {
728 if (AllocatePagesFunc
== NULL
) {
729 AllocatePagesFunc
= AllocatePageTableMemory
;
733 // Make sure that the page table is changeable.
735 IsWpEnabled
= IsReadOnlyPageWriteProtected ();
737 DisableReadOnlyPageWriteProtect ();
741 // Below logic is to check 2M/4K page to make sure we donot waist memory.
743 Status
= EFI_SUCCESS
;
744 while (Length
!= 0) {
745 PageEntry
= GetPageTableEntry (&CurrentPagingContext
, BaseAddress
, &PageAttribute
);
746 if (PageEntry
== NULL
) {
747 Status
= RETURN_UNSUPPORTED
;
750 PageEntryLength
= PageAttributeToLength (PageAttribute
);
751 SplitAttribute
= NeedSplitPage (BaseAddress
, Length
, PageEntry
, PageAttribute
);
752 if (SplitAttribute
== PageNone
) {
753 ConvertPageEntryAttribute (&CurrentPagingContext
, PageEntry
, Attributes
, PageAction
, &IsEntryModified
);
754 if (IsEntryModified
) {
755 if (IsModified
!= NULL
) {
760 // Convert success, move to next
762 BaseAddress
+= PageEntryLength
;
763 Length
-= PageEntryLength
;
765 if (AllocatePagesFunc
== NULL
) {
766 Status
= RETURN_UNSUPPORTED
;
769 Status
= SplitPage (PageEntry
, PageAttribute
, SplitAttribute
, AllocatePagesFunc
);
770 if (RETURN_ERROR (Status
)) {
771 Status
= RETURN_UNSUPPORTED
;
774 if (IsSplitted
!= NULL
) {
777 if (IsModified
!= NULL
) {
781 // Just split current page
782 // Convert success in next around
789 // Restore page table write protection, if any.
792 EnableReadOnlyPageWriteProtect ();
798 This function assigns the page attributes for the memory region specified by BaseAddress and
799 Length from their current attributes to the attributes specified by Attributes.
801 Caller should make sure BaseAddress and Length is at page boundary.
803 Caller need guarentee the TPL <= TPL_NOTIFY, if there is split page request.
805 @param[in] PagingContext The paging context. NULL means get page table from current CPU context.
806 @param[in] BaseAddress The physical address that is the start address of a memory region.
807 @param[in] Length The size in bytes of the memory region.
808 @param[in] Attributes The bit mask of attributes to set for the memory region.
809 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
810 NULL mean page split is unsupported.
812 @retval RETURN_SUCCESS The attributes were cleared for the memory region.
813 @retval RETURN_ACCESS_DENIED The attributes for the memory resource range specified by
814 BaseAddress and Length cannot be modified.
815 @retval RETURN_INVALID_PARAMETER Length is zero.
816 Attributes specified an illegal combination of attributes that
817 cannot be set together.
818 @retval RETURN_OUT_OF_RESOURCES There are not enough system resources to modify the attributes of
819 the memory resource range.
820 @retval RETURN_UNSUPPORTED The processor does not support one or more bytes of the memory
821 resource range specified by BaseAddress and Length.
822 The bit mask of attributes is not support for the memory resource
823 range specified by BaseAddress and Length.
827 AssignMemoryPageAttributes (
828 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext OPTIONAL
,
829 IN PHYSICAL_ADDRESS BaseAddress
,
831 IN UINT64 Attributes
,
832 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc OPTIONAL
835 RETURN_STATUS Status
;
839 // DEBUG((DEBUG_INFO, "AssignMemoryPageAttributes: 0x%lx - 0x%lx (0x%lx)\n", BaseAddress, Length, Attributes));
840 Status
= ConvertMemoryPageAttributes (PagingContext
, BaseAddress
, Length
, Attributes
, PageActionAssign
, AllocatePagesFunc
, &IsSplitted
, &IsModified
);
841 if (!EFI_ERROR(Status
)) {
842 if ((PagingContext
== NULL
) && IsModified
) {
844 // Flush TLB as last step.
846 // Note: Since APs will always init CR3 register in HLT loop mode or do
847 // TLB flush in MWAIT loop mode, there's no need to flush TLB for them
858 Check if Execute Disable feature is enabled or not.
861 IsExecuteDisableEnabled (
865 MSR_CORE_IA32_EFER_REGISTER MsrEfer
;
867 MsrEfer
.Uint64
= AsmReadMsr64 (MSR_IA32_EFER
);
868 return (MsrEfer
.Bits
.NXE
== 1);
872 Update GCD memory space attributes according to current page table setup.
875 RefreshGcdMemoryAttributesFromPaging (
880 UINTN NumberOfDescriptors
;
881 EFI_GCD_MEMORY_SPACE_DESCRIPTOR
*MemorySpaceMap
;
882 PAGE_TABLE_LIB_PAGING_CONTEXT PagingContext
;
883 PAGE_ATTRIBUTE PageAttribute
;
886 UINT64 MemorySpaceLength
;
889 UINT64 PageStartAddress
;
892 UINT64 NewAttributes
;
896 // Assuming that memory space map returned is sorted already; otherwise sort
897 // them in the order of lowest address to highest address.
899 Status
= gDS
->GetMemorySpaceMap (&NumberOfDescriptors
, &MemorySpaceMap
);
900 ASSERT_EFI_ERROR (Status
);
902 GetCurrentPagingContext (&PagingContext
);
909 if (IsExecuteDisableEnabled ()) {
910 Capabilities
= EFI_MEMORY_RO
| EFI_MEMORY_RP
| EFI_MEMORY_XP
;
912 Capabilities
= EFI_MEMORY_RO
| EFI_MEMORY_RP
;
915 for (Index
= 0; Index
< NumberOfDescriptors
; Index
++) {
916 if (MemorySpaceMap
[Index
].GcdMemoryType
== EfiGcdMemoryTypeNonExistent
) {
921 // Sync the actual paging related capabilities back to GCD service first.
922 // As a side effect (good one), this can also help to avoid unnecessary
923 // memory map entries due to the different capabilities of the same type
924 // memory, such as multiple RT_CODE and RT_DATA entries in memory map,
925 // which could cause boot failure of some old Linux distro (before v4.3).
927 Status
= gDS
->SetMemorySpaceCapabilities (
928 MemorySpaceMap
[Index
].BaseAddress
,
929 MemorySpaceMap
[Index
].Length
,
930 MemorySpaceMap
[Index
].Capabilities
| Capabilities
932 if (EFI_ERROR (Status
)) {
934 // If we cannot udpate the capabilities, we cannot update its
935 // attributes either. So just simply skip current block of memory.
939 "Failed to update capability: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
940 (UINT64
)Index
, MemorySpaceMap
[Index
].BaseAddress
,
941 MemorySpaceMap
[Index
].BaseAddress
+ MemorySpaceMap
[Index
].Length
- 1,
942 MemorySpaceMap
[Index
].Capabilities
,
943 MemorySpaceMap
[Index
].Capabilities
| Capabilities
948 if (MemorySpaceMap
[Index
].BaseAddress
>= (BaseAddress
+ PageLength
)) {
950 // Current memory space starts at a new page. Resetting PageLength will
951 // trigger a retrieval of page attributes at new address.
956 // In case current memory space is not adjacent to last one
958 PageLength
-= (MemorySpaceMap
[Index
].BaseAddress
- BaseAddress
);
962 // Sync actual page attributes to GCD
964 BaseAddress
= MemorySpaceMap
[Index
].BaseAddress
;
965 MemorySpaceLength
= MemorySpaceMap
[Index
].Length
;
966 while (MemorySpaceLength
> 0) {
967 if (PageLength
== 0) {
968 PageEntry
= GetPageTableEntry (&PagingContext
, BaseAddress
, &PageAttribute
);
969 if (PageEntry
== NULL
) {
974 // Note current memory space might start in the middle of a page
976 PageStartAddress
= (*PageEntry
) & (UINT64
)PageAttributeToMask(PageAttribute
);
977 PageLength
= PageAttributeToLength (PageAttribute
) - (BaseAddress
- PageStartAddress
);
978 Attributes
= GetAttributesFromPageEntry (PageEntry
);
981 Length
= MIN (PageLength
, MemorySpaceLength
);
982 if (Attributes
!= (MemorySpaceMap
[Index
].Attributes
&
983 EFI_MEMORY_PAGETYPE_MASK
)) {
984 NewAttributes
= (MemorySpaceMap
[Index
].Attributes
&
985 ~EFI_MEMORY_PAGETYPE_MASK
) | Attributes
;
986 Status
= gDS
->SetMemorySpaceAttributes (
991 ASSERT_EFI_ERROR (Status
);
994 "Updated memory space attribute: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
995 (UINT64
)Index
, BaseAddress
, BaseAddress
+ Length
- 1,
996 MemorySpaceMap
[Index
].Attributes
,
1001 PageLength
-= Length
;
1002 MemorySpaceLength
-= Length
;
1003 BaseAddress
+= Length
;
1007 FreePool (MemorySpaceMap
);
1011 Initialize a buffer pool for page table use only.
1013 To reduce the potential split operation on page table, the pages reserved for
1014 page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
1015 at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
1016 initialized with number of pages greater than or equal to the given PoolPages.
1018 Once the pages in the pool are used up, this method should be called again to
1019 reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. Usually this won't happen
1022 @param[in] PoolPages The least page number of the pool to be created.
1024 @retval TRUE The pool is initialized successfully.
1025 @retval FALSE The memory is out of resource.
1028 InitializePageTablePool (
1036 // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
1039 PoolPages
+= 1; // Add one page for header.
1040 PoolPages
= ((PoolPages
- 1) / PAGE_TABLE_POOL_UNIT_PAGES
+ 1) *
1041 PAGE_TABLE_POOL_UNIT_PAGES
;
1042 Buffer
= AllocateAlignedPages (PoolPages
, PAGE_TABLE_POOL_ALIGNMENT
);
1043 if (Buffer
== NULL
) {
1044 DEBUG ((DEBUG_ERROR
, "ERROR: Out of aligned pages\r\n"));
1049 // Link all pools into a list for easier track later.
1051 if (mPageTablePool
== NULL
) {
1052 mPageTablePool
= Buffer
;
1053 mPageTablePool
->NextPool
= mPageTablePool
;
1055 ((PAGE_TABLE_POOL
*)Buffer
)->NextPool
= mPageTablePool
->NextPool
;
1056 mPageTablePool
->NextPool
= Buffer
;
1057 mPageTablePool
= Buffer
;
1061 // Reserve one page for pool header.
1063 mPageTablePool
->FreePages
= PoolPages
- 1;
1064 mPageTablePool
->Offset
= EFI_PAGES_TO_SIZE (1);
1067 // Mark the whole pool pages as read-only.
1069 ConvertMemoryPageAttributes (
1071 (PHYSICAL_ADDRESS
)(UINTN
)Buffer
,
1072 EFI_PAGES_TO_SIZE (PoolPages
),
1075 AllocatePageTableMemory
,
1079 ASSERT (IsModified
== TRUE
);
1085 This API provides a way to allocate memory for page table.
1087 This API can be called more than once to allocate memory for page tables.
1089 Allocates the number of 4KB pages and returns a pointer to the allocated
1090 buffer. The buffer returned is aligned on a 4KB boundary.
1092 If Pages is 0, then NULL is returned.
1093 If there is not enough memory remaining to satisfy the request, then NULL is
1096 @param Pages The number of 4 KB pages to allocate.
1098 @return A pointer to the allocated buffer or NULL if allocation fails.
1103 AllocatePageTableMemory (
1114 // Renew the pool if necessary.
1116 if (mPageTablePool
== NULL
||
1117 Pages
> mPageTablePool
->FreePages
) {
1118 if (!InitializePageTablePool (Pages
)) {
1123 Buffer
= (UINT8
*)mPageTablePool
+ mPageTablePool
->Offset
;
1125 mPageTablePool
->Offset
+= EFI_PAGES_TO_SIZE (Pages
);
1126 mPageTablePool
->FreePages
-= Pages
;
1132 Initialize the Page Table lib.
1135 InitializePageTableLib (
1139 PAGE_TABLE_LIB_PAGING_CONTEXT CurrentPagingContext
;
1141 GetCurrentPagingContext (&CurrentPagingContext
);
1144 // Reserve memory of page tables for future uses, if paging is enabled.
1146 if (CurrentPagingContext
.ContextData
.X64
.PageTableBase
!= 0 &&
1147 (CurrentPagingContext
.ContextData
.Ia32
.Attributes
&
1148 PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) != 0) {
1149 DisableReadOnlyPageWriteProtect ();
1150 InitializePageTablePool (1);
1151 EnableReadOnlyPageWriteProtect ();
1154 DEBUG ((DEBUG_INFO
, "CurrentPagingContext:\n", CurrentPagingContext
.MachineType
));
1155 DEBUG ((DEBUG_INFO
, " MachineType - 0x%x\n", CurrentPagingContext
.MachineType
));
1156 DEBUG ((DEBUG_INFO
, " PageTableBase - 0x%x\n", CurrentPagingContext
.ContextData
.X64
.PageTableBase
));
1157 DEBUG ((DEBUG_INFO
, " Attributes - 0x%x\n", CurrentPagingContext
.ContextData
.X64
.Attributes
));