/** @file\r
Page Fault (#PF) handler for X64 processors\r
\r
-Copyright (c) 2009 - 2016, Intel Corporation. All rights reserved.<BR>\r
+Copyright (c) 2009 - 2019, Intel Corporation. All rights reserved.<BR>\r
+Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>\r
+\r
This program and the accompanying materials\r
are licensed and made available under the terms and conditions of the BSD License\r
which accompanies this distribution. The full text of the license may be found at\r
\r
#define PAGE_TABLE_PAGES 8\r
#define ACC_MAX_BIT BIT3\r
+\r
LIST_ENTRY mPagePool = INITIALIZE_LIST_HEAD_VARIABLE (mPagePool);\r
BOOLEAN m1GPageTableSupport = FALSE;\r
-UINT8 mPhysicalAddressBits;\r
BOOLEAN mCpuSmmStaticPageTable;\r
\r
+/**\r
+ Disable CET.\r
+**/\r
+VOID\r
+EFIAPI\r
+DisableCet (\r
+ VOID\r
+ );\r
+\r
+/**\r
+ Enable CET.\r
+**/\r
+VOID\r
+EFIAPI\r
+EnableCet (\r
+ VOID\r
+ );\r
+\r
/**\r
Check if 1-GByte pages is supported by processor or not.\r
\r
//\r
// Each PML4 entry points to a page of Page Directory Pointer entries.\r
//\r
- PageDirectoryPointerEntry = (UINT64 *) ((*PageMapLevel4Entry) & gPhyMask);\r
+ PageDirectoryPointerEntry = (UINT64 *) ((*PageMapLevel4Entry) & ~mAddressEncMask & gPhyMask);\r
if (PageDirectoryPointerEntry == NULL) {\r
PageDirectoryPointerEntry = AllocatePageTableMemory (1);\r
ASSERT(PageDirectoryPointerEntry != NULL);\r
ZeroMem (PageDirectoryPointerEntry, EFI_PAGES_TO_SIZE(1));\r
\r
- *PageMapLevel4Entry = ((UINTN)PageDirectoryPointerEntry & gPhyMask) | PAGE_ATTRIBUTE_BITS;\r
+ *PageMapLevel4Entry = (UINT64)(UINTN)PageDirectoryPointerEntry | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
}\r
\r
if (m1GPageTableSupport) {\r
//\r
// Fill in the Page Directory entries\r
//\r
- *PageDirectory1GEntry = (PageAddress & gPhyMask) | IA32_PG_PS | PAGE_ATTRIBUTE_BITS;\r
+ *PageDirectory1GEntry = PageAddress | mAddressEncMask | IA32_PG_PS | PAGE_ATTRIBUTE_BITS;\r
}\r
} else {\r
PageAddress = BASE_4GB;\r
// Each Directory Pointer entries points to a page of Page Directory entires.\r
// So allocate space for them and fill them in in the IndexOfPageDirectoryEntries loop.\r
//\r
- PageDirectoryEntry = (UINT64 *) ((*PageDirectoryPointerEntry) & gPhyMask);\r
+ PageDirectoryEntry = (UINT64 *) ((*PageDirectoryPointerEntry) & ~mAddressEncMask & gPhyMask);\r
if (PageDirectoryEntry == NULL) {\r
PageDirectoryEntry = AllocatePageTableMemory (1);\r
ASSERT(PageDirectoryEntry != NULL);\r
//\r
// Fill in a Page Directory Pointer Entries\r
//\r
- *PageDirectoryPointerEntry = (UINT64)(UINTN)PageDirectoryEntry | PAGE_ATTRIBUTE_BITS;\r
+ *PageDirectoryPointerEntry = (UINT64)(UINTN)PageDirectoryEntry | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
}\r
\r
for (IndexOfPageDirectoryEntries = 0; IndexOfPageDirectoryEntries < 512; IndexOfPageDirectoryEntries++, PageDirectoryEntry++, PageAddress += SIZE_2MB) {\r
//\r
// Fill in the Page Directory entries\r
//\r
- *PageDirectoryEntry = (UINT64)PageAddress | IA32_PG_PS | PAGE_ATTRIBUTE_BITS;\r
+ *PageDirectoryEntry = PageAddress | mAddressEncMask | IA32_PG_PS | PAGE_ATTRIBUTE_BITS;\r
}\r
}\r
}\r
//\r
PTEntry = (UINT64*)AllocatePageTableMemory (1);\r
ASSERT (PTEntry != NULL);\r
- *PTEntry = Pages | PAGE_ATTRIBUTE_BITS;\r
+ *PTEntry = Pages | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
ZeroMem (PTEntry + 1, EFI_PAGE_SIZE - sizeof (*PTEntry));\r
\r
//\r
}\r
}\r
\r
- if (FeaturePcdGet (PcdCpuSmmProfileEnable)) {\r
+ if (FeaturePcdGet (PcdCpuSmmProfileEnable) ||\r
+ HEAP_GUARD_NONSTOP_MODE ||\r
+ NULL_DETECTION_NONSTOP_MODE) {\r
//\r
// Set own Page Fault entry instead of the default one, because SMM Profile\r
// feature depends on IRET instruction to do Single Step\r
//\r
continue;\r
}\r
- Pdpt = (UINT64*)(UINTN)(Pml4[Pml4Index] & gPhyMask);\r
+ Pdpt = (UINT64*)(UINTN)(Pml4[Pml4Index] & ~mAddressEncMask & gPhyMask);\r
PML4EIgnore = FALSE;\r
for (PdptIndex = 0; PdptIndex < EFI_PAGE_SIZE / sizeof (*Pdpt); PdptIndex++) {\r
if ((Pdpt[PdptIndex] & IA32_PG_P) == 0 || (Pdpt[PdptIndex] & IA32_PG_PMNT) != 0) {\r
// we will not check PML4 entry more\r
//\r
PML4EIgnore = TRUE;\r
- Pdt = (UINT64*)(UINTN)(Pdpt[PdptIndex] & gPhyMask);\r
+ Pdt = (UINT64*)(UINTN)(Pdpt[PdptIndex] & ~mAddressEncMask & gPhyMask);\r
PDPTEIgnore = FALSE;\r
for (PdtIndex = 0; PdtIndex < EFI_PAGE_SIZE / sizeof(*Pdt); PdtIndex++) {\r
if ((Pdt[PdtIndex] & IA32_PG_P) == 0 || (Pdt[PdtIndex] & IA32_PG_PMNT) != 0) {\r
//\r
// Secondly, insert the page pointed by this entry into page pool and clear this entry\r
//\r
- InsertTailList (&mPagePool, (LIST_ENTRY*)(UINTN)(*ReleasePageAddress & gPhyMask));\r
+ InsertTailList (&mPagePool, (LIST_ENTRY*)(UINTN)(*ReleasePageAddress & ~mAddressEncMask & gPhyMask));\r
*ReleasePageAddress = 0;\r
\r
//\r
//\r
// If 4 KByte Page Table is released, check the PDPT entry\r
//\r
- Pdpt = (UINT64*)(UINTN)(Pml4[MinPml4] & gPhyMask);\r
+ Pdpt = (UINT64*)(UINTN)(Pml4[MinPml4] & ~mAddressEncMask & gPhyMask);\r
SubEntriesNum = GetSubEntriesNum(Pdpt + MinPdpt);\r
if (SubEntriesNum == 0) {\r
//\r
// Release the empty Page Directory table if there was no more 4 KByte Page Table entry\r
// clear the Page directory entry\r
//\r
- InsertTailList (&mPagePool, (LIST_ENTRY*)(UINTN)(Pdpt[MinPdpt] & gPhyMask));\r
+ InsertTailList (&mPagePool, (LIST_ENTRY*)(UINTN)(Pdpt[MinPdpt] & ~mAddressEncMask & gPhyMask));\r
Pdpt[MinPdpt] = 0;\r
//\r
// Go on checking the PML4 table\r
// Release the empty PML4 table if there was no more 1G KByte Page Table entry\r
// clear the Page directory entry\r
//\r
- InsertTailList (&mPagePool, (LIST_ENTRY*)(UINTN)(Pml4[MinPml4] & gPhyMask));\r
+ InsertTailList (&mPagePool, (LIST_ENTRY*)(UINTN)(Pml4[MinPml4] & ~mAddressEncMask & gPhyMask));\r
Pml4[MinPml4] = 0;\r
MinPdpt = (UINTN)-1;\r
continue;\r
//\r
// If the entry is not present, allocate one page from page pool for it\r
//\r
- PageTable[PTIndex] = AllocPage () | PAGE_ATTRIBUTE_BITS;\r
+ PageTable[PTIndex] = AllocPage () | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
} else {\r
//\r
// Save the upper entry address\r
//\r
PageTable[PTIndex] |= (UINT64)IA32_PG_A;\r
SetAccNum (PageTable + PTIndex, 7);\r
- PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & gPhyMask);\r
+ PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & gPhyMask);\r
}\r
\r
PTIndex = BitFieldRead64 (PFAddress, StartBit, StartBit + 8);\r
//\r
// Fill the new entry\r
//\r
- PageTable[PTIndex] = (PFAddress & gPhyMask & ~((1ull << EndBit) - 1)) |\r
+ PageTable[PTIndex] = ((PFAddress | mAddressEncMask) & gPhyMask & ~((1ull << EndBit) - 1)) |\r
PageAttribute | IA32_PG_A | PAGE_ATTRIBUTE_BITS;\r
if (UpperEntry != NULL) {\r
SetSubEntriesNum (UpperEntry, GetSubEntriesNum (UpperEntry) + 1);\r
VOID\r
EFIAPI\r
SmiPFHandler (\r
- IN EFI_EXCEPTION_TYPE InterruptType,\r
- IN EFI_SYSTEM_CONTEXT SystemContext\r
+ IN EFI_EXCEPTION_TYPE InterruptType,\r
+ IN EFI_SYSTEM_CONTEXT SystemContext\r
)\r
{\r
UINTN PFAddress;\r
+ UINTN GuardPageAddress;\r
+ UINTN CpuIndex;\r
\r
ASSERT (InterruptType == EXCEPT_IA32_PAGE_FAULT);\r
\r
PFAddress = AsmReadCr2 ();\r
\r
if (mCpuSmmStaticPageTable && (PFAddress >= LShiftU64 (1, (mPhysicalAddressBits - 1)))) {\r
+ DumpCpuContext (InterruptType, SystemContext);\r
DEBUG ((DEBUG_ERROR, "Do not support address 0x%lx by processor!\n", PFAddress));\r
CpuDeadLoop ();\r
+ goto Exit;\r
}\r
\r
//\r
- // If a page fault occurs in SMRAM range, it should be in a SMM stack guard page.\r
+ // If a page fault occurs in SMRAM range, it might be in a SMM stack guard page,\r
+ // or SMM page protection violation.\r
//\r
- if ((FeaturePcdGet (PcdCpuSmmStackGuard)) &&\r
- (PFAddress >= mCpuHotPlugData.SmrrBase) &&\r
+ if ((PFAddress >= mCpuHotPlugData.SmrrBase) &&\r
(PFAddress < (mCpuHotPlugData.SmrrBase + mCpuHotPlugData.SmrrSize))) {\r
- DEBUG ((DEBUG_ERROR, "SMM stack overflow!\n"));\r
+ DumpCpuContext (InterruptType, SystemContext);\r
+ CpuIndex = GetCpuIndex ();\r
+ GuardPageAddress = (mSmmStackArrayBase + EFI_PAGE_SIZE + CpuIndex * mSmmStackSize);\r
+ if ((FeaturePcdGet (PcdCpuSmmStackGuard)) &&\r
+ (PFAddress >= GuardPageAddress) &&\r
+ (PFAddress < (GuardPageAddress + EFI_PAGE_SIZE))) {\r
+ DEBUG ((DEBUG_ERROR, "SMM stack overflow!\n"));\r
+ } else {\r
+ if ((SystemContext.SystemContextX64->ExceptionData & IA32_PF_EC_ID) != 0) {\r
+ DEBUG ((DEBUG_ERROR, "SMM exception at execution (0x%lx)\n", PFAddress));\r
+ DEBUG_CODE (\r
+ DumpModuleInfoByIp (*(UINTN *)(UINTN)SystemContext.SystemContextX64->Rsp);\r
+ );\r
+ } else {\r
+ DEBUG ((DEBUG_ERROR, "SMM exception at access (0x%lx)\n", PFAddress));\r
+ DEBUG_CODE (\r
+ DumpModuleInfoByIp ((UINTN)SystemContext.SystemContextX64->Rip);\r
+ );\r
+ }\r
+\r
+ if (HEAP_GUARD_NONSTOP_MODE) {\r
+ GuardPagePFHandler (SystemContext.SystemContextX64->ExceptionData);\r
+ goto Exit;\r
+ }\r
+ }\r
CpuDeadLoop ();\r
+ goto Exit;\r
}\r
\r
//\r
- // If a page fault occurs in SMM range\r
+ // If a page fault occurs in non-SMRAM range.\r
//\r
if ((PFAddress < mCpuHotPlugData.SmrrBase) ||\r
(PFAddress >= mCpuHotPlugData.SmrrBase + mCpuHotPlugData.SmrrSize)) {\r
if ((SystemContext.SystemContextX64->ExceptionData & IA32_PF_EC_ID) != 0) {\r
+ DumpCpuContext (InterruptType, SystemContext);\r
DEBUG ((DEBUG_ERROR, "Code executed on IP(0x%lx) out of SMM range after SMM is locked!\n", PFAddress));\r
DEBUG_CODE (\r
DumpModuleInfoByIp (*(UINTN *)(UINTN)SystemContext.SystemContextX64->Rsp);\r
);\r
CpuDeadLoop ();\r
+ goto Exit;\r
+ }\r
+\r
+ //\r
+ // If NULL pointer was just accessed\r
+ //\r
+ if ((PcdGet8 (PcdNullPointerDetectionPropertyMask) & BIT1) != 0 &&\r
+ (PFAddress < EFI_PAGE_SIZE)) {\r
+ DumpCpuContext (InterruptType, SystemContext);\r
+ DEBUG ((DEBUG_ERROR, "!!! NULL pointer access !!!\n"));\r
+ DEBUG_CODE (\r
+ DumpModuleInfoByIp ((UINTN)SystemContext.SystemContextX64->Rip);\r
+ );\r
+\r
+ if (NULL_DETECTION_NONSTOP_MODE) {\r
+ GuardPagePFHandler (SystemContext.SystemContextX64->ExceptionData);\r
+ goto Exit;\r
+ }\r
+\r
+ CpuDeadLoop ();\r
+ goto Exit;\r
+ }\r
+\r
+ if (mCpuSmmStaticPageTable && IsSmmCommBufferForbiddenAddress (PFAddress)) {\r
+ DumpCpuContext (InterruptType, SystemContext);\r
+ DEBUG ((DEBUG_ERROR, "Access SMM communication forbidden address (0x%lx)!\n", PFAddress));\r
+ DEBUG_CODE (\r
+ DumpModuleInfoByIp ((UINTN)SystemContext.SystemContextX64->Rip);\r
+ );\r
+ CpuDeadLoop ();\r
+ goto Exit;\r
}\r
}\r
\r
SmiDefaultPFHandler ();\r
}\r
\r
+Exit:\r
ReleaseSpinLock (mPFLock);\r
}\r
\r
UINT64 *L4PageTable;\r
BOOLEAN IsSplitted;\r
BOOLEAN PageTableSplitted;\r
+ BOOLEAN CetEnabled;\r
+\r
+ //\r
+ // Don't do this if\r
+ // - no static page table; or\r
+ // - SMM heap guard feature enabled; or\r
+ // BIT2: SMM page guard enabled\r
+ // BIT3: SMM pool guard enabled\r
+ // - SMM profile feature enabled\r
+ //\r
+ if (!mCpuSmmStaticPageTable ||\r
+ ((PcdGet8 (PcdHeapGuardPropertyMask) & (BIT3 | BIT2)) != 0) ||\r
+ FeaturePcdGet (PcdCpuSmmProfileEnable)) {\r
+ //\r
+ // Static paging and heap guard could not be enabled at the same time.\r
+ //\r
+ ASSERT (!(mCpuSmmStaticPageTable &&\r
+ (PcdGet8 (PcdHeapGuardPropertyMask) & (BIT3 | BIT2)) != 0));\r
\r
- if (!mCpuSmmStaticPageTable) {\r
+ //\r
+ // Static paging and SMM profile could not be enabled at the same time.\r
+ //\r
+ ASSERT (!(mCpuSmmStaticPageTable && FeaturePcdGet (PcdCpuSmmProfileEnable)));\r
return ;\r
}\r
\r
// Disable write protection, because we need mark page table to be write protected.\r
// We need *write* page table memory, to mark itself to be *read only*.\r
//\r
+ CetEnabled = ((AsmReadCr4() & CR4_CET_ENABLE) != 0) ? TRUE : FALSE;\r
+ if (CetEnabled) {\r
+ //\r
+ // CET must be disabled if WP is disabled.\r
+ //\r
+ DisableCet();\r
+ }\r
AsmWriteCr0 (AsmReadCr0() & ~CR0_WP);\r
\r
do {\r
PageTableSplitted = (PageTableSplitted || IsSplitted);\r
\r
for (Index4 = 0; Index4 < SIZE_4KB/sizeof(UINT64); Index4++) {\r
- L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & PAGING_4K_ADDRESS_MASK_64);\r
+ L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & ~mAddressEncMask & PAGING_4K_ADDRESS_MASK_64);\r
if (L3PageTable == NULL) {\r
continue;\r
}\r
// 1G\r
continue;\r
}\r
- L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & PAGING_4K_ADDRESS_MASK_64);\r
+ L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & ~mAddressEncMask & PAGING_4K_ADDRESS_MASK_64);\r
if (L2PageTable == NULL) {\r
continue;\r
}\r
// 2M\r
continue;\r
}\r
- L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & PAGING_4K_ADDRESS_MASK_64);\r
+ L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & ~mAddressEncMask & PAGING_4K_ADDRESS_MASK_64);\r
if (L1PageTable == NULL) {\r
continue;\r
}\r
// Enable write protection, after page table updated.\r
//\r
AsmWriteCr0 (AsmReadCr0() | CR0_WP);\r
+ if (CetEnabled) {\r
+ //\r
+ // re-enable CET.\r
+ //\r
+ EnableCet();\r
+ }\r
\r
return ;\r
}\r
projects / mirror_edk2.git / blobdiff