2 Page table management support.
4 Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
7 This program and the accompanying materials
8 are licensed and made available under the terms and conditions of the BSD License
9 which accompanies this distribution. The full text of the license may be found at
10 http://opensource.org/licenses/bsd-license.php
12 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
13 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
19 #include <Library/BaseLib.h>
20 #include <Library/CpuLib.h>
21 #include <Library/BaseMemoryLib.h>
22 #include <Library/MemoryAllocationLib.h>
23 #include <Library/DebugLib.h>
24 #include <Library/UefiBootServicesTableLib.h>
25 #include <Protocol/MpService.h>
28 #include "CpuPageTable.h"
33 #define IA32_PG_P BIT0
34 #define IA32_PG_RW BIT1
35 #define IA32_PG_U BIT2
36 #define IA32_PG_WT BIT3
37 #define IA32_PG_CD BIT4
38 #define IA32_PG_A BIT5
39 #define IA32_PG_D BIT6
40 #define IA32_PG_PS BIT7
41 #define IA32_PG_PAT_2M BIT12
42 #define IA32_PG_PAT_4K IA32_PG_PS
43 #define IA32_PG_PMNT BIT62
44 #define IA32_PG_NX BIT63
46 #define PAGE_ATTRIBUTE_BITS (IA32_PG_D | IA32_PG_A | IA32_PG_U | IA32_PG_RW | IA32_PG_P)
48 // Bits 1, 2, 5, 6 are reserved in the IA32 PAE PDPTE
49 // X64 PAE PDPTE does not have such restriction
51 #define IA32_PAE_PDPTE_ATTRIBUTE_BITS (IA32_PG_P)
53 #define PAGE_PROGATE_BITS (IA32_PG_NX | PAGE_ATTRIBUTE_BITS)
55 #define PAGING_4K_MASK 0xFFF
56 #define PAGING_2M_MASK 0x1FFFFF
57 #define PAGING_1G_MASK 0x3FFFFFFF
59 #define PAGING_PAE_INDEX_MASK 0x1FF
61 #define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull
62 #define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull
63 #define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull
73 PAGE_ATTRIBUTE Attribute
;
76 } PAGE_ATTRIBUTE_TABLE
;
84 PAGE_ATTRIBUTE_TABLE mPageAttributeTable
[] = {
85 {Page4K
, SIZE_4KB
, PAGING_4K_ADDRESS_MASK_64
},
86 {Page2M
, SIZE_2MB
, PAGING_2M_ADDRESS_MASK_64
},
87 {Page1G
, SIZE_1GB
, PAGING_1G_ADDRESS_MASK_64
},
90 PAGE_TABLE_POOL
*mPageTablePool
= NULL
;
93 Return current paging context.
95 @param[in,out] PagingContext The paging context.
98 GetCurrentPagingContext (
99 IN OUT PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
105 ZeroMem(PagingContext
, sizeof(*PagingContext
));
106 if (sizeof(UINTN
) == sizeof(UINT64
)) {
107 PagingContext
->MachineType
= IMAGE_FILE_MACHINE_X64
;
109 PagingContext
->MachineType
= IMAGE_FILE_MACHINE_I386
;
111 if ((AsmReadCr0 () & BIT31
) != 0) {
112 PagingContext
->ContextData
.X64
.PageTableBase
= (AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64
);
114 PagingContext
->ContextData
.X64
.PageTableBase
= 0;
117 if ((AsmReadCr4 () & BIT4
) != 0) {
118 PagingContext
->ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PSE
;
120 if ((AsmReadCr4 () & BIT5
) != 0) {
121 PagingContext
->ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
;
123 if ((AsmReadCr0 () & BIT16
) != 0) {
124 PagingContext
->ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_WP_ENABLE
;
127 AsmCpuid (0x80000000, &RegEax
, NULL
, NULL
, NULL
);
128 if (RegEax
> 0x80000000) {
129 AsmCpuid (0x80000001, NULL
, NULL
, NULL
, &RegEdx
);
130 if ((RegEdx
& BIT20
) != 0) {
132 if ((AsmReadMsr64 (0xC0000080) & BIT11
) != 0) {
134 PagingContext
->ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED
;
137 if ((RegEdx
& BIT26
) != 0) {
138 PagingContext
->ContextData
.Ia32
.Attributes
|= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAGE_1G_SUPPORT
;
144 Return length according to page attributes.
146 @param[in] PageAttributes The page attribute of the page entry.
148 @return The length of page entry.
151 PageAttributeToLength (
152 IN PAGE_ATTRIBUTE PageAttribute
156 for (Index
= 0; Index
< sizeof(mPageAttributeTable
)/sizeof(mPageAttributeTable
[0]); Index
++) {
157 if (PageAttribute
== mPageAttributeTable
[Index
].Attribute
) {
158 return (UINTN
)mPageAttributeTable
[Index
].Length
;
165 Return address mask according to page attributes.
167 @param[in] PageAttributes The page attribute of the page entry.
169 @return The address mask of page entry.
172 PageAttributeToMask (
173 IN PAGE_ATTRIBUTE PageAttribute
177 for (Index
= 0; Index
< sizeof(mPageAttributeTable
)/sizeof(mPageAttributeTable
[0]); Index
++) {
178 if (PageAttribute
== mPageAttributeTable
[Index
].Attribute
) {
179 return (UINTN
)mPageAttributeTable
[Index
].AddressMask
;
186 Return page table entry to match the address.
188 @param[in] PagingContext The paging context.
189 @param[in] Address The address to be checked.
190 @param[out] PageAttributes The page attribute of the page entry.
192 @return The page entry.
196 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
,
197 IN PHYSICAL_ADDRESS Address
,
198 OUT PAGE_ATTRIBUTE
*PageAttribute
209 UINT64 AddressEncMask
;
211 ASSERT (PagingContext
!= NULL
);
213 Index4
= ((UINTN
)RShiftU64 (Address
, 39)) & PAGING_PAE_INDEX_MASK
;
214 Index3
= ((UINTN
)Address
>> 30) & PAGING_PAE_INDEX_MASK
;
215 Index2
= ((UINTN
)Address
>> 21) & PAGING_PAE_INDEX_MASK
;
216 Index1
= ((UINTN
)Address
>> 12) & PAGING_PAE_INDEX_MASK
;
218 // Make sure AddressEncMask is contained to smallest supported address field.
220 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
222 if (PagingContext
->MachineType
== IMAGE_FILE_MACHINE_X64
) {
223 L4PageTable
= (UINT64
*)(UINTN
)PagingContext
->ContextData
.X64
.PageTableBase
;
224 if (L4PageTable
[Index4
] == 0) {
225 *PageAttribute
= PageNone
;
229 L3PageTable
= (UINT64
*)(UINTN
)(L4PageTable
[Index4
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
231 ASSERT((PagingContext
->ContextData
.Ia32
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) != 0);
232 L3PageTable
= (UINT64
*)(UINTN
)PagingContext
->ContextData
.Ia32
.PageTableBase
;
234 if (L3PageTable
[Index3
] == 0) {
235 *PageAttribute
= PageNone
;
238 if ((L3PageTable
[Index3
] & IA32_PG_PS
) != 0) {
240 *PageAttribute
= Page1G
;
241 return &L3PageTable
[Index3
];
244 L2PageTable
= (UINT64
*)(UINTN
)(L3PageTable
[Index3
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
245 if (L2PageTable
[Index2
] == 0) {
246 *PageAttribute
= PageNone
;
249 if ((L2PageTable
[Index2
] & IA32_PG_PS
) != 0) {
251 *PageAttribute
= Page2M
;
252 return &L2PageTable
[Index2
];
256 L1PageTable
= (UINT64
*)(UINTN
)(L2PageTable
[Index2
] & ~AddressEncMask
& PAGING_4K_ADDRESS_MASK_64
);
257 if ((L1PageTable
[Index1
] == 0) && (Address
!= 0)) {
258 *PageAttribute
= PageNone
;
261 *PageAttribute
= Page4K
;
262 return &L1PageTable
[Index1
];
266 Return memory attributes of page entry.
268 @param[in] PageEntry The page entry.
270 @return Memory attributes of page entry.
273 GetAttributesFromPageEntry (
279 if ((*PageEntry
& IA32_PG_P
) == 0) {
280 Attributes
|= EFI_MEMORY_RP
;
282 if ((*PageEntry
& IA32_PG_RW
) == 0) {
283 Attributes
|= EFI_MEMORY_RO
;
285 if ((*PageEntry
& IA32_PG_NX
) != 0) {
286 Attributes
|= EFI_MEMORY_XP
;
292 Modify memory attributes of page entry.
294 @param[in] PagingContext The paging context.
295 @param[in] PageEntry The page entry.
296 @param[in] Attributes The bit mask of attributes to modify for the memory region.
297 @param[in] PageAction The page action.
298 @param[out] IsModified TRUE means page table modified. FALSE means page table not modified.
301 ConvertPageEntryAttribute (
302 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext
,
303 IN UINT64
*PageEntry
,
304 IN UINT64 Attributes
,
305 IN PAGE_ACTION PageAction
,
306 OUT BOOLEAN
*IsModified
309 UINT64 CurrentPageEntry
;
312 CurrentPageEntry
= *PageEntry
;
313 NewPageEntry
= CurrentPageEntry
;
314 if ((Attributes
& EFI_MEMORY_RP
) != 0) {
315 switch (PageAction
) {
316 case PageActionAssign
:
318 NewPageEntry
&= ~(UINT64
)IA32_PG_P
;
320 case PageActionClear
:
321 NewPageEntry
|= IA32_PG_P
;
325 switch (PageAction
) {
326 case PageActionAssign
:
327 NewPageEntry
|= IA32_PG_P
;
330 case PageActionClear
:
334 if ((Attributes
& EFI_MEMORY_RO
) != 0) {
335 switch (PageAction
) {
336 case PageActionAssign
:
338 NewPageEntry
&= ~(UINT64
)IA32_PG_RW
;
340 case PageActionClear
:
341 NewPageEntry
|= IA32_PG_RW
;
345 switch (PageAction
) {
346 case PageActionAssign
:
347 NewPageEntry
|= IA32_PG_RW
;
350 case PageActionClear
:
354 if ((PagingContext
->ContextData
.Ia32
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED
) != 0) {
355 if ((Attributes
& EFI_MEMORY_XP
) != 0) {
356 switch (PageAction
) {
357 case PageActionAssign
:
359 NewPageEntry
|= IA32_PG_NX
;
361 case PageActionClear
:
362 NewPageEntry
&= ~IA32_PG_NX
;
366 switch (PageAction
) {
367 case PageActionAssign
:
368 NewPageEntry
&= ~IA32_PG_NX
;
371 case PageActionClear
:
376 *PageEntry
= NewPageEntry
;
377 if (CurrentPageEntry
!= NewPageEntry
) {
379 DEBUG ((DEBUG_VERBOSE
, "ConvertPageEntryAttribute 0x%lx", CurrentPageEntry
));
380 DEBUG ((DEBUG_VERBOSE
, "->0x%lx\n", NewPageEntry
));
387 This function returns if there is need to split page entry.
389 @param[in] BaseAddress The base address to be checked.
390 @param[in] Length The length to be checked.
391 @param[in] PageEntry The page entry to be checked.
392 @param[in] PageAttribute The page attribute of the page entry.
394 @retval SplitAttributes on if there is need to split page entry.
398 IN PHYSICAL_ADDRESS BaseAddress
,
400 IN UINT64
*PageEntry
,
401 IN PAGE_ATTRIBUTE PageAttribute
404 UINT64 PageEntryLength
;
406 PageEntryLength
= PageAttributeToLength (PageAttribute
);
408 if (((BaseAddress
& (PageEntryLength
- 1)) == 0) && (Length
>= PageEntryLength
)) {
412 if (((BaseAddress
& PAGING_2M_MASK
) != 0) || (Length
< SIZE_2MB
)) {
420 This function splits one page entry to small page entries.
422 @param[in] PageEntry The page entry to be splitted.
423 @param[in] PageAttribute The page attribute of the page entry.
424 @param[in] SplitAttribute How to split the page entry.
425 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
427 @retval RETURN_SUCCESS The page entry is splitted.
428 @retval RETURN_UNSUPPORTED The page entry does not support to be splitted.
429 @retval RETURN_OUT_OF_RESOURCES No resource to split page entry.
433 IN UINT64
*PageEntry
,
434 IN PAGE_ATTRIBUTE PageAttribute
,
435 IN PAGE_ATTRIBUTE SplitAttribute
,
436 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc
440 UINT64
*NewPageEntry
;
442 UINT64 AddressEncMask
;
444 ASSERT (PageAttribute
== Page2M
|| PageAttribute
== Page1G
);
446 ASSERT (AllocatePagesFunc
!= NULL
);
448 // Make sure AddressEncMask is contained to smallest supported address field.
450 AddressEncMask
= PcdGet64 (PcdPteMemoryEncryptionAddressOrMask
) & PAGING_1G_ADDRESS_MASK_64
;
452 if (PageAttribute
== Page2M
) {
456 ASSERT (SplitAttribute
== Page4K
);
457 if (SplitAttribute
== Page4K
) {
458 NewPageEntry
= AllocatePagesFunc (1);
459 DEBUG ((DEBUG_INFO
, "Split - 0x%x\n", NewPageEntry
));
460 if (NewPageEntry
== NULL
) {
461 return RETURN_OUT_OF_RESOURCES
;
463 BaseAddress
= *PageEntry
& ~AddressEncMask
& PAGING_2M_ADDRESS_MASK_64
;
464 for (Index
= 0; Index
< SIZE_4KB
/ sizeof(UINT64
); Index
++) {
465 NewPageEntry
[Index
] = (BaseAddress
+ SIZE_4KB
* Index
) | AddressEncMask
| ((*PageEntry
) & PAGE_PROGATE_BITS
);
467 (*PageEntry
) = (UINT64
)(UINTN
)NewPageEntry
| AddressEncMask
| ((*PageEntry
) & PAGE_ATTRIBUTE_BITS
);
468 return RETURN_SUCCESS
;
470 return RETURN_UNSUPPORTED
;
472 } else if (PageAttribute
== Page1G
) {
475 // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
477 ASSERT (SplitAttribute
== Page2M
|| SplitAttribute
== Page4K
);
478 if ((SplitAttribute
== Page2M
|| SplitAttribute
== Page4K
)) {
479 NewPageEntry
= AllocatePagesFunc (1);
480 DEBUG ((DEBUG_INFO
, "Split - 0x%x\n", NewPageEntry
));
481 if (NewPageEntry
== NULL
) {
482 return RETURN_OUT_OF_RESOURCES
;
484 BaseAddress
= *PageEntry
& ~AddressEncMask
& PAGING_1G_ADDRESS_MASK_64
;
485 for (Index
= 0; Index
< SIZE_4KB
/ sizeof(UINT64
); Index
++) {
486 NewPageEntry
[Index
] = (BaseAddress
+ SIZE_2MB
* Index
) | AddressEncMask
| IA32_PG_PS
| ((*PageEntry
) & PAGE_PROGATE_BITS
);
488 (*PageEntry
) = (UINT64
)(UINTN
)NewPageEntry
| AddressEncMask
| ((*PageEntry
) & PAGE_ATTRIBUTE_BITS
);
489 return RETURN_SUCCESS
;
491 return RETURN_UNSUPPORTED
;
494 return RETURN_UNSUPPORTED
;
499 Check the WP status in CR0 register. This bit is used to lock or unlock write
500 access to pages marked as read-only.
502 @retval TRUE Write protection is enabled.
503 @retval FALSE Write protection is disabled.
506 IsReadOnlyPageWriteProtected (
510 return ((AsmReadCr0 () & BIT16
) != 0);
514 Disable Write Protect on pages marked as read-only.
517 DisableReadOnlyPageWriteProtect (
521 AsmWriteCr0 (AsmReadCr0() & ~BIT16
);
525 Enable Write Protect on pages marked as read-only.
528 EnableReadOnlyPageWriteProtect (
532 AsmWriteCr0 (AsmReadCr0() | BIT16
);
536 This function modifies the page attributes for the memory region specified by BaseAddress and
537 Length from their current attributes to the attributes specified by Attributes.
539 Caller should make sure BaseAddress and Length is at page boundary.
541 @param[in] PagingContext The paging context. NULL means get page table from current CPU context.
542 @param[in] BaseAddress The physical address that is the start address of a memory region.
543 @param[in] Length The size in bytes of the memory region.
544 @param[in] Attributes The bit mask of attributes to modify for the memory region.
545 @param[in] PageAction The page action.
546 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
547 NULL mean page split is unsupported.
548 @param[out] IsSplitted TRUE means page table splitted. FALSE means page table not splitted.
549 @param[out] IsModified TRUE means page table modified. FALSE means page table not modified.
551 @retval RETURN_SUCCESS The attributes were modified for the memory region.
552 @retval RETURN_ACCESS_DENIED The attributes for the memory resource range specified by
553 BaseAddress and Length cannot be modified.
554 @retval RETURN_INVALID_PARAMETER Length is zero.
555 Attributes specified an illegal combination of attributes that
556 cannot be set together.
557 @retval RETURN_OUT_OF_RESOURCES There are not enough system resources to modify the attributes of
558 the memory resource range.
559 @retval RETURN_UNSUPPORTED The processor does not support one or more bytes of the memory
560 resource range specified by BaseAddress and Length.
561 The bit mask of attributes is not support for the memory resource
562 range specified by BaseAddress and Length.
565 ConvertMemoryPageAttributes (
566 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext OPTIONAL
,
567 IN PHYSICAL_ADDRESS BaseAddress
,
569 IN UINT64 Attributes
,
570 IN PAGE_ACTION PageAction
,
571 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc OPTIONAL
,
572 OUT BOOLEAN
*IsSplitted
, OPTIONAL
573 OUT BOOLEAN
*IsModified OPTIONAL
576 PAGE_TABLE_LIB_PAGING_CONTEXT CurrentPagingContext
;
578 PAGE_ATTRIBUTE PageAttribute
;
579 UINTN PageEntryLength
;
580 PAGE_ATTRIBUTE SplitAttribute
;
581 RETURN_STATUS Status
;
582 BOOLEAN IsEntryModified
;
585 if ((BaseAddress
& (SIZE_4KB
- 1)) != 0) {
586 DEBUG ((DEBUG_ERROR
, "BaseAddress(0x%lx) is not aligned!\n", BaseAddress
));
587 return EFI_UNSUPPORTED
;
589 if ((Length
& (SIZE_4KB
- 1)) != 0) {
590 DEBUG ((DEBUG_ERROR
, "Length(0x%lx) is not aligned!\n", Length
));
591 return EFI_UNSUPPORTED
;
594 DEBUG ((DEBUG_ERROR
, "Length is 0!\n"));
595 return RETURN_INVALID_PARAMETER
;
598 if ((Attributes
& ~(EFI_MEMORY_RP
| EFI_MEMORY_RO
| EFI_MEMORY_XP
)) != 0) {
599 DEBUG ((DEBUG_ERROR
, "Attributes(0x%lx) has unsupported bit\n", Attributes
));
600 return EFI_UNSUPPORTED
;
603 if (PagingContext
== NULL
) {
604 GetCurrentPagingContext (&CurrentPagingContext
);
606 CopyMem (&CurrentPagingContext
, PagingContext
, sizeof(CurrentPagingContext
));
608 switch(CurrentPagingContext
.MachineType
) {
609 case IMAGE_FILE_MACHINE_I386
:
610 if (CurrentPagingContext
.ContextData
.Ia32
.PageTableBase
== 0) {
611 if (Attributes
== 0) {
614 DEBUG ((DEBUG_ERROR
, "PageTable is 0!\n"));
615 return EFI_UNSUPPORTED
;
618 if ((CurrentPagingContext
.ContextData
.Ia32
.Attributes
& PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) == 0) {
619 DEBUG ((DEBUG_ERROR
, "Non-PAE Paging!\n"));
620 return EFI_UNSUPPORTED
;
622 if ((BaseAddress
+ Length
) > BASE_4GB
) {
623 DEBUG ((DEBUG_ERROR
, "Beyond 4GB memory in 32-bit mode!\n"));
624 return EFI_UNSUPPORTED
;
627 case IMAGE_FILE_MACHINE_X64
:
628 ASSERT (CurrentPagingContext
.ContextData
.X64
.PageTableBase
!= 0);
632 return EFI_UNSUPPORTED
;
636 // DEBUG ((DEBUG_ERROR, "ConvertMemoryPageAttributes(%x) - %016lx, %016lx, %02lx\n", IsSet, BaseAddress, Length, Attributes));
638 if (IsSplitted
!= NULL
) {
641 if (IsModified
!= NULL
) {
644 if (AllocatePagesFunc
== NULL
) {
645 AllocatePagesFunc
= AllocatePageTableMemory
;
649 // Make sure that the page table is changeable.
651 IsWpEnabled
= IsReadOnlyPageWriteProtected ();
653 DisableReadOnlyPageWriteProtect ();
657 // Below logic is to check 2M/4K page to make sure we donot waist memory.
659 Status
= EFI_SUCCESS
;
660 while (Length
!= 0) {
661 PageEntry
= GetPageTableEntry (&CurrentPagingContext
, BaseAddress
, &PageAttribute
);
662 if (PageEntry
== NULL
) {
663 Status
= RETURN_UNSUPPORTED
;
666 PageEntryLength
= PageAttributeToLength (PageAttribute
);
667 SplitAttribute
= NeedSplitPage (BaseAddress
, Length
, PageEntry
, PageAttribute
);
668 if (SplitAttribute
== PageNone
) {
669 ConvertPageEntryAttribute (&CurrentPagingContext
, PageEntry
, Attributes
, PageAction
, &IsEntryModified
);
670 if (IsEntryModified
) {
671 if (IsModified
!= NULL
) {
676 // Convert success, move to next
678 BaseAddress
+= PageEntryLength
;
679 Length
-= PageEntryLength
;
681 if (AllocatePagesFunc
== NULL
) {
682 Status
= RETURN_UNSUPPORTED
;
685 Status
= SplitPage (PageEntry
, PageAttribute
, SplitAttribute
, AllocatePagesFunc
);
686 if (RETURN_ERROR (Status
)) {
687 Status
= RETURN_UNSUPPORTED
;
690 if (IsSplitted
!= NULL
) {
693 if (IsModified
!= NULL
) {
697 // Just split current page
698 // Convert success in next around
705 // Restore page table write protection, if any.
708 EnableReadOnlyPageWriteProtect ();
714 This function assigns the page attributes for the memory region specified by BaseAddress and
715 Length from their current attributes to the attributes specified by Attributes.
717 Caller should make sure BaseAddress and Length is at page boundary.
719 Caller need guarentee the TPL <= TPL_NOTIFY, if there is split page request.
721 @param[in] PagingContext The paging context. NULL means get page table from current CPU context.
722 @param[in] BaseAddress The physical address that is the start address of a memory region.
723 @param[in] Length The size in bytes of the memory region.
724 @param[in] Attributes The bit mask of attributes to set for the memory region.
725 @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
726 NULL mean page split is unsupported.
728 @retval RETURN_SUCCESS The attributes were cleared for the memory region.
729 @retval RETURN_ACCESS_DENIED The attributes for the memory resource range specified by
730 BaseAddress and Length cannot be modified.
731 @retval RETURN_INVALID_PARAMETER Length is zero.
732 Attributes specified an illegal combination of attributes that
733 cannot be set together.
734 @retval RETURN_OUT_OF_RESOURCES There are not enough system resources to modify the attributes of
735 the memory resource range.
736 @retval RETURN_UNSUPPORTED The processor does not support one or more bytes of the memory
737 resource range specified by BaseAddress and Length.
738 The bit mask of attributes is not support for the memory resource
739 range specified by BaseAddress and Length.
743 AssignMemoryPageAttributes (
744 IN PAGE_TABLE_LIB_PAGING_CONTEXT
*PagingContext OPTIONAL
,
745 IN PHYSICAL_ADDRESS BaseAddress
,
747 IN UINT64 Attributes
,
748 IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc OPTIONAL
751 RETURN_STATUS Status
;
755 // DEBUG((DEBUG_INFO, "AssignMemoryPageAttributes: 0x%lx - 0x%lx (0x%lx)\n", BaseAddress, Length, Attributes));
756 Status
= ConvertMemoryPageAttributes (PagingContext
, BaseAddress
, Length
, Attributes
, PageActionAssign
, AllocatePagesFunc
, &IsSplitted
, &IsModified
);
757 if (!EFI_ERROR(Status
)) {
758 if ((PagingContext
== NULL
) && IsModified
) {
760 // Flush TLB as last step.
762 // Note: Since APs will always init CR3 register in HLT loop mode or do
763 // TLB flush in MWAIT loop mode, there's no need to flush TLB for them
774 Check if Execute Disable feature is enabled or not.
777 IsExecuteDisableEnabled (
781 MSR_CORE_IA32_EFER_REGISTER MsrEfer
;
783 MsrEfer
.Uint64
= AsmReadMsr64 (MSR_IA32_EFER
);
784 return (MsrEfer
.Bits
.NXE
== 1);
788 Update GCD memory space attributes according to current page table setup.
791 RefreshGcdMemoryAttributesFromPaging (
796 UINTN NumberOfDescriptors
;
797 EFI_GCD_MEMORY_SPACE_DESCRIPTOR
*MemorySpaceMap
;
798 PAGE_TABLE_LIB_PAGING_CONTEXT PagingContext
;
799 PAGE_ATTRIBUTE PageAttribute
;
802 UINT64 MemorySpaceLength
;
805 UINT64 PageStartAddress
;
808 UINT64 NewAttributes
;
812 // Assuming that memory space map returned is sorted already; otherwise sort
813 // them in the order of lowest address to highest address.
815 Status
= gDS
->GetMemorySpaceMap (&NumberOfDescriptors
, &MemorySpaceMap
);
816 ASSERT_EFI_ERROR (Status
);
818 GetCurrentPagingContext (&PagingContext
);
825 if (IsExecuteDisableEnabled ()) {
826 Capabilities
= EFI_MEMORY_RO
| EFI_MEMORY_RP
| EFI_MEMORY_XP
;
828 Capabilities
= EFI_MEMORY_RO
| EFI_MEMORY_RP
;
831 for (Index
= 0; Index
< NumberOfDescriptors
; Index
++) {
832 if (MemorySpaceMap
[Index
].GcdMemoryType
== EfiGcdMemoryTypeNonExistent
) {
837 // Sync the actual paging related capabilities back to GCD service first.
838 // As a side effect (good one), this can also help to avoid unnecessary
839 // memory map entries due to the different capabilities of the same type
840 // memory, such as multiple RT_CODE and RT_DATA entries in memory map,
841 // which could cause boot failure of some old Linux distro (before v4.3).
843 Status
= gDS
->SetMemorySpaceCapabilities (
844 MemorySpaceMap
[Index
].BaseAddress
,
845 MemorySpaceMap
[Index
].Length
,
846 MemorySpaceMap
[Index
].Capabilities
| Capabilities
848 if (EFI_ERROR (Status
)) {
850 // If we cannot udpate the capabilities, we cannot update its
851 // attributes either. So just simply skip current block of memory.
855 "Failed to update capability: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
856 (UINT64
)Index
, MemorySpaceMap
[Index
].BaseAddress
,
857 MemorySpaceMap
[Index
].BaseAddress
+ MemorySpaceMap
[Index
].Length
- 1,
858 MemorySpaceMap
[Index
].Capabilities
,
859 MemorySpaceMap
[Index
].Capabilities
| Capabilities
864 if (MemorySpaceMap
[Index
].BaseAddress
>= (BaseAddress
+ PageLength
)) {
866 // Current memory space starts at a new page. Resetting PageLength will
867 // trigger a retrieval of page attributes at new address.
872 // In case current memory space is not adjacent to last one
874 PageLength
-= (MemorySpaceMap
[Index
].BaseAddress
- BaseAddress
);
878 // Sync actual page attributes to GCD
880 BaseAddress
= MemorySpaceMap
[Index
].BaseAddress
;
881 MemorySpaceLength
= MemorySpaceMap
[Index
].Length
;
882 while (MemorySpaceLength
> 0) {
883 if (PageLength
== 0) {
884 PageEntry
= GetPageTableEntry (&PagingContext
, BaseAddress
, &PageAttribute
);
885 if (PageEntry
== NULL
) {
890 // Note current memory space might start in the middle of a page
892 PageStartAddress
= (*PageEntry
) & (UINT64
)PageAttributeToMask(PageAttribute
);
893 PageLength
= PageAttributeToLength (PageAttribute
) - (BaseAddress
- PageStartAddress
);
894 Attributes
= GetAttributesFromPageEntry (PageEntry
);
897 Length
= MIN (PageLength
, MemorySpaceLength
);
898 if (Attributes
!= (MemorySpaceMap
[Index
].Attributes
&
899 EFI_MEMORY_PAGETYPE_MASK
)) {
900 NewAttributes
= (MemorySpaceMap
[Index
].Attributes
&
901 ~EFI_MEMORY_PAGETYPE_MASK
) | Attributes
;
902 Status
= gDS
->SetMemorySpaceAttributes (
907 ASSERT_EFI_ERROR (Status
);
910 "Updated memory space attribute: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
911 (UINT64
)Index
, BaseAddress
, BaseAddress
+ Length
- 1,
912 MemorySpaceMap
[Index
].Attributes
,
917 PageLength
-= Length
;
918 MemorySpaceLength
-= Length
;
919 BaseAddress
+= Length
;
923 FreePool (MemorySpaceMap
);
927 Initialize a buffer pool for page table use only.
929 To reduce the potential split operation on page table, the pages reserved for
930 page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
931 at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
932 initialized with number of pages greater than or equal to the given PoolPages.
934 Once the pages in the pool are used up, this method should be called again to
935 reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. Usually this won't happen
938 @param[in] PoolPages The least page number of the pool to be created.
940 @retval TRUE The pool is initialized successfully.
941 @retval FALSE The memory is out of resource.
944 InitializePageTablePool (
952 // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
955 PoolPages
+= 1; // Add one page for header.
956 PoolPages
= ((PoolPages
- 1) / PAGE_TABLE_POOL_UNIT_PAGES
+ 1) *
957 PAGE_TABLE_POOL_UNIT_PAGES
;
958 Buffer
= AllocateAlignedPages (PoolPages
, PAGE_TABLE_POOL_ALIGNMENT
);
959 if (Buffer
== NULL
) {
960 DEBUG ((DEBUG_ERROR
, "ERROR: Out of aligned pages\r\n"));
965 // Link all pools into a list for easier track later.
967 if (mPageTablePool
== NULL
) {
968 mPageTablePool
= Buffer
;
969 mPageTablePool
->NextPool
= mPageTablePool
;
971 ((PAGE_TABLE_POOL
*)Buffer
)->NextPool
= mPageTablePool
->NextPool
;
972 mPageTablePool
->NextPool
= Buffer
;
973 mPageTablePool
= Buffer
;
977 // Reserve one page for pool header.
979 mPageTablePool
->FreePages
= PoolPages
- 1;
980 mPageTablePool
->Offset
= EFI_PAGES_TO_SIZE (1);
983 // Mark the whole pool pages as read-only.
985 ConvertMemoryPageAttributes (
987 (PHYSICAL_ADDRESS
)(UINTN
)Buffer
,
988 EFI_PAGES_TO_SIZE (PoolPages
),
991 AllocatePageTableMemory
,
995 ASSERT (IsModified
== TRUE
);
1001 This API provides a way to allocate memory for page table.
1003 This API can be called more than once to allocate memory for page tables.
1005 Allocates the number of 4KB pages and returns a pointer to the allocated
1006 buffer. The buffer returned is aligned on a 4KB boundary.
1008 If Pages is 0, then NULL is returned.
1009 If there is not enough memory remaining to satisfy the request, then NULL is
1012 @param Pages The number of 4 KB pages to allocate.
1014 @return A pointer to the allocated buffer or NULL if allocation fails.
1019 AllocatePageTableMemory (
1030 // Renew the pool if necessary.
1032 if (mPageTablePool
== NULL
||
1033 Pages
> mPageTablePool
->FreePages
) {
1034 if (!InitializePageTablePool (Pages
)) {
1039 Buffer
= (UINT8
*)mPageTablePool
+ mPageTablePool
->Offset
;
1041 mPageTablePool
->Offset
+= EFI_PAGES_TO_SIZE (Pages
);
1042 mPageTablePool
->FreePages
-= Pages
;
1048 Initialize the Page Table lib.
1051 InitializePageTableLib (
1055 PAGE_TABLE_LIB_PAGING_CONTEXT CurrentPagingContext
;
1057 GetCurrentPagingContext (&CurrentPagingContext
);
1060 // Reserve memory of page tables for future uses, if paging is enabled.
1062 if (CurrentPagingContext
.ContextData
.X64
.PageTableBase
!= 0 &&
1063 (CurrentPagingContext
.ContextData
.Ia32
.Attributes
&
1064 PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE
) != 0) {
1065 DisableReadOnlyPageWriteProtect ();
1066 InitializePageTablePool (1);
1067 EnableReadOnlyPageWriteProtect ();
1070 DEBUG ((DEBUG_INFO
, "CurrentPagingContext:\n", CurrentPagingContext
.MachineType
));
1071 DEBUG ((DEBUG_INFO
, " MachineType - 0x%x\n", CurrentPagingContext
.MachineType
));
1072 DEBUG ((DEBUG_INFO
, " PageTableBase - 0x%x\n", CurrentPagingContext
.ContextData
.X64
.PageTableBase
));
1073 DEBUG ((DEBUG_INFO
, " Attributes - 0x%x\n", CurrentPagingContext
.ContextData
.X64
.Attributes
));