[mirror_edk2.git] / UefiCpuPkg / PiSmmCpuDxeSmm / X64 / SmmProfileArch.c

/** @file\r
X64 processor specific functions to enable SMM profile.\r
\r
Copyright (c) 2012 - 2019, Intel Corporation. All rights reserved.<BR>\r
Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>\r
\r
SPDX-License-Identifier: BSD-2-Clause-Patent\r
\r
**/\r
\r
#include "PiSmmCpuDxeSmm.h"\r
#include "SmmProfileInternal.h"\r
\r
//\r
// Current page index.\r
//\r
UINTN                     mPFPageIndex;\r
\r
//\r
// Pool for dynamically creating page table in page fault handler.\r
//\r
UINT64                    mPFPageBuffer;\r
\r
//\r
// Store the uplink information for each page being used.\r
//\r
UINT64                    *mPFPageUplink[MAX_PF_PAGE_COUNT];\r
\r
/**\r
  Create SMM page table for S3 path.\r
\r
**/\r
VOID\r
InitSmmS3Cr3 (\r
  VOID\r
  )\r
{\r
  EFI_PHYSICAL_ADDRESS              Pages;\r
  UINT64                            *PTEntry;\r
\r
  //\r
  // Generate PAE page table for the first 4GB memory space\r
  //\r
  Pages = Gen4GPageTable (FALSE);\r
\r
  //\r
  // Fill Page-Table-Level4 (PML4) entry\r
  //\r
  PTEntry = (UINT64*)AllocatePageTableMemory (1);\r
  ASSERT (PTEntry != NULL);\r
  *PTEntry = Pages | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
  ZeroMem (PTEntry + 1, EFI_PAGE_SIZE - sizeof (*PTEntry));\r
\r
  //\r
  // Return the address of PML4 (to set CR3)\r
  //\r
  mSmmS3ResumeState->SmmS3Cr3 = (UINT32)(UINTN)PTEntry;\r
\r
  return ;\r
}\r
\r
/**\r
  Allocate pages for creating 4KB-page based on 2MB-page when page fault happens.\r
\r
**/\r
VOID\r
InitPagesForPFHandler (\r
  VOID\r
  )\r
{\r
  VOID          *Address;\r
\r
  //\r
  // Pre-Allocate memory for page fault handler\r
  //\r
  Address = NULL;\r
  Address = AllocatePages (MAX_PF_PAGE_COUNT);\r
  ASSERT (Address != NULL);\r
\r
  mPFPageBuffer =  (UINT64)(UINTN) Address;\r
  mPFPageIndex = 0;\r
  ZeroMem ((VOID *) (UINTN) mPFPageBuffer, EFI_PAGE_SIZE * MAX_PF_PAGE_COUNT);\r
  ZeroMem (mPFPageUplink, sizeof (mPFPageUplink));\r
\r
  return;\r
}\r
\r
/**\r
  Allocate one page for creating 4KB-page based on 2MB-page.\r
\r
  @param  Uplink   The address of Page-Directory entry.\r
\r
**/\r
VOID\r
AcquirePage (\r
  UINT64          *Uplink\r
  )\r
{\r
  UINT64          Address;\r
\r
  //\r
  // Get the buffer\r
  //\r
  Address = mPFPageBuffer + EFI_PAGES_TO_SIZE (mPFPageIndex);\r
  ZeroMem ((VOID *) (UINTN) Address, EFI_PAGE_SIZE);\r
\r
  //\r
  // Cut the previous uplink if it exists and wasn't overwritten\r
  //\r
  if ((mPFPageUplink[mPFPageIndex] != NULL) && ((*mPFPageUplink[mPFPageIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK) == Address)) {\r
    *mPFPageUplink[mPFPageIndex] = 0;\r
  }\r
\r
  //\r
  // Link & Record the current uplink\r
  //\r
  *Uplink = Address | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
  mPFPageUplink[mPFPageIndex] = Uplink;\r
\r
  mPFPageIndex = (mPFPageIndex + 1) % MAX_PF_PAGE_COUNT;\r
}\r
\r
/**\r
  Update page table to map the memory correctly in order to make the instruction\r
  which caused page fault execute successfully. And it also save the original page\r
  table to be restored in single-step exception.\r
\r
  @param  PageTable           PageTable Address.\r
  @param  PFAddress           The memory address which caused page fault exception.\r
  @param  CpuIndex            The index of the processor.\r
  @param  ErrorCode           The Error code of exception.\r
  @param  IsValidPFAddress    The flag indicates if SMM profile data need be added.\r
\r
**/\r
VOID\r
RestorePageTableAbove4G (\r
  UINT64        *PageTable,\r
  UINT64        PFAddress,\r
  UINTN         CpuIndex,\r
  UINTN         ErrorCode,\r
  BOOLEAN       *IsValidPFAddress\r
  )\r
{\r
  UINTN         PTIndex;\r
  UINT64        Address;\r
  BOOLEAN       Nx;\r
  BOOLEAN       Existed;\r
  UINTN         Index;\r
  UINTN         PFIndex;\r
  IA32_CR4      Cr4;\r
  BOOLEAN       Enable5LevelPaging;\r
\r
  ASSERT ((PageTable != NULL) && (IsValidPFAddress != NULL));\r
\r
  Cr4.UintN = AsmReadCr4 ();\r
  Enable5LevelPaging = (BOOLEAN) (Cr4.Bits.LA57 == 1);\r
\r
  //\r
  // If page fault address is 4GB above.\r
  //\r
\r
  //\r
  // Check if page fault address has existed in page table.\r
  // If it exists in page table but page fault is generated,\r
  // there are 2 possible reasons: 1. present flag is set to 0; 2. instruction fetch in protected memory range.\r
  //\r
  Existed = FALSE;\r
  PageTable = (UINT64*)(AsmReadCr3 () & PHYSICAL_ADDRESS_MASK);\r
  PTIndex = 0;\r
  if (Enable5LevelPaging) {\r
    PTIndex = BitFieldRead64 (PFAddress, 48, 56);\r
  }\r
  if ((!Enable5LevelPaging) || ((PageTable[PTIndex] & IA32_PG_P) != 0)) {\r
    // PML5E\r
    if (Enable5LevelPaging) {\r
      PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
    }\r
    PTIndex = BitFieldRead64 (PFAddress, 39, 47);\r
    if ((PageTable[PTIndex] & IA32_PG_P) != 0) {\r
      // PML4E\r
      PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
      PTIndex = BitFieldRead64 (PFAddress, 30, 38);\r
      if ((PageTable[PTIndex] & IA32_PG_P) != 0) {\r
        // PDPTE\r
        PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
        PTIndex = BitFieldRead64 (PFAddress, 21, 29);\r
        // PD\r
        if ((PageTable[PTIndex] & IA32_PG_PS) != 0) {\r
          //\r
          // 2MB page\r
          //\r
          Address = (UINT64)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
          if ((Address & ~((1ull << 21) - 1)) == ((PFAddress & PHYSICAL_ADDRESS_MASK & ~((1ull << 21) - 1)))) {\r
            Existed = TRUE;\r
          }\r
        } else {\r
          //\r
          // 4KB page\r
          //\r
          PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask& PHYSICAL_ADDRESS_MASK);\r
          if (PageTable != 0) {\r
            //\r
            // When there is a valid entry to map to 4KB page, need not create a new entry to map 2MB.\r
            //\r
            PTIndex = BitFieldRead64 (PFAddress, 12, 20);\r
            Address = (UINT64)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
            if ((Address & ~((1ull << 12) - 1)) == (PFAddress & PHYSICAL_ADDRESS_MASK & ~((1ull << 12) - 1))) {\r
              Existed = TRUE;\r
            }\r
          }\r
        }\r
      }\r
    }\r
  }\r
\r
  //\r
  // If page entry does not existed in page table at all, create a new entry.\r
  //\r
  if (!Existed) {\r
\r
    if (IsAddressValid (PFAddress, &Nx)) {\r
      //\r
      // If page fault address above 4GB is in protected range but it causes a page fault exception,\r
      // Will create a page entry for this page fault address, make page table entry as present/rw and execution-disable.\r
      // this access is not saved into SMM profile data.\r
      //\r
      *IsValidPFAddress = TRUE;\r
    }\r
\r
    //\r
    // Create one entry in page table for page fault address.\r
    //\r
    SmiDefaultPFHandler ();\r
    //\r
    // Find the page table entry created just now.\r
    //\r
    PageTable = (UINT64*)(AsmReadCr3 () & PHYSICAL_ADDRESS_MASK);\r
    PFAddress = AsmReadCr2 ();\r
    // PML5E\r
    if (Enable5LevelPaging) {\r
      PTIndex = BitFieldRead64 (PFAddress, 48, 56);\r
      PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
    }\r
    // PML4E\r
    PTIndex = BitFieldRead64 (PFAddress, 39, 47);\r
    PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
    // PDPTE\r
    PTIndex = BitFieldRead64 (PFAddress, 30, 38);\r
    PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
    // PD\r
    PTIndex = BitFieldRead64 (PFAddress, 21, 29);\r
    Address = PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK;\r
    //\r
    // Check if 2MB-page entry need be changed to 4KB-page entry.\r
    //\r
    if (IsAddressSplit (Address)) {\r
      AcquirePage (&PageTable[PTIndex]);\r
\r
      // PTE\r
      PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
      for (Index = 0; Index < 512; Index++) {\r
        PageTable[Index] = Address | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
        if (!IsAddressValid (Address, &Nx)) {\r
          PageTable[Index] = PageTable[Index] & (INTN)(INT32)(~PAGE_ATTRIBUTE_BITS);\r
        }\r
        if (Nx && mXdSupported) {\r
          PageTable[Index] = PageTable[Index] | IA32_PG_NX;\r
        }\r
        if (Address == (PFAddress & PHYSICAL_ADDRESS_MASK & ~((1ull << 12) - 1))) {\r
          PTIndex = Index;\r
        }\r
        Address += SIZE_4KB;\r
      } // end for PT\r
    } else {\r
      //\r
      // Update 2MB page entry.\r
      //\r
      if (!IsAddressValid (Address, &Nx)) {\r
        //\r
        // Patch to remove present flag and rw flag.\r
        //\r
        PageTable[PTIndex] = PageTable[PTIndex] & (INTN)(INT32)(~PAGE_ATTRIBUTE_BITS);\r
      }\r
      //\r
      // Set XD bit to 1\r
      //\r
      if (Nx && mXdSupported) {\r
        PageTable[PTIndex] = PageTable[PTIndex] | IA32_PG_NX;\r
      }\r
    }\r
  }\r
\r
  //\r
  // Record old entries with non-present status\r
  // Old entries include the memory which instruction is at and the memory which instruction access.\r
  //\r
  //\r
  ASSERT (mPFEntryCount[CpuIndex] < MAX_PF_ENTRY_COUNT);\r
  if (mPFEntryCount[CpuIndex] < MAX_PF_ENTRY_COUNT) {\r
    PFIndex = mPFEntryCount[CpuIndex];\r
    mLastPFEntryValue[CpuIndex][PFIndex]   = PageTable[PTIndex];\r
    mLastPFEntryPointer[CpuIndex][PFIndex] = &PageTable[PTIndex];\r
    mPFEntryCount[CpuIndex]++;\r
  }\r
\r
  //\r
  // Add present flag or clear XD flag to make page fault handler succeed.\r
  //\r
  PageTable[PTIndex] |= (UINT64)(PAGE_ATTRIBUTE_BITS);\r
  if ((ErrorCode & IA32_PF_EC_ID) != 0) {\r
    //\r
    // If page fault is caused by instruction fetch, clear XD bit in the entry.\r
    //\r
    PageTable[PTIndex] &= ~IA32_PG_NX;\r
  }\r
\r
  return;\r
}\r
\r
/**\r
  Clear TF in FLAGS.\r
\r
  @param  SystemContext    A pointer to the processor context when\r
                           the interrupt occurred on the processor.\r
\r
**/\r
VOID\r
ClearTrapFlag (\r
  IN OUT EFI_SYSTEM_CONTEXT   SystemContext\r
  )\r
{\r
  SystemContext.SystemContextX64->Rflags &= (UINTN) ~BIT8;\r
}\r
Commit	Line	Data
427e3573 MK	1	/** @file\r
	2	X64 processor specific functions to enable SMM profile.\r
	3	\r
7365eb2c	4	Copyright (c) 2012 - 2019, Intel Corporation. All rights reserved.<BR>\r
241f9149 LD	5	Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>\r
241f9149 LD	6	\r
0acd8697	7	SPDX-License-Identifier: BSD-2-Clause-Patent\r
427e3573 MK	8	\r
	9	**/\r
	10	\r
	11	#include "PiSmmCpuDxeSmm.h"\r
	12	#include "SmmProfileInternal.h"\r
	13	\r
	14	//\r
	15	// Current page index.\r
	16	//\r
	17	UINTN mPFPageIndex;\r
	18	\r
	19	//\r
	20	// Pool for dynamically creating page table in page fault handler.\r
	21	//\r
	22	UINT64 mPFPageBuffer;\r
	23	\r
	24	//\r
	25	// Store the uplink information for each page being used.\r
	26	//\r
	27	UINT64 *mPFPageUplink[MAX_PF_PAGE_COUNT];\r
	28	\r
	29	/**\r
	30	Create SMM page table for S3 path.\r
	31	\r
	32	**/\r
	33	VOID\r
	34	InitSmmS3Cr3 (\r
	35	VOID\r
	36	)\r
	37	{\r
	38	EFI_PHYSICAL_ADDRESS Pages;\r
	39	UINT64 *PTEntry;\r
	40	\r
	41	//\r
	42	// Generate PAE page table for the first 4GB memory space\r
	43	//\r
717fb604	44	Pages = Gen4GPageTable (FALSE);\r
427e3573 MK	45	\r
	46	//\r
	47	// Fill Page-Table-Level4 (PML4) entry\r
	48	//\r
717fb604 JY	49	PTEntry = (UINT64*)AllocatePageTableMemory (1);\r
717fb604 JY	50	ASSERT (PTEntry != NULL);\r
241f9149	51	*PTEntry = Pages \| mAddressEncMask \| PAGE_ATTRIBUTE_BITS;\r
427e3573 MK	52	ZeroMem (PTEntry + 1, EFI_PAGE_SIZE - sizeof (*PTEntry));\r
	53	\r
	54	//\r
	55	// Return the address of PML4 (to set CR3)\r
	56	//\r
	57	mSmmS3ResumeState->SmmS3Cr3 = (UINT32)(UINTN)PTEntry;\r
	58	\r
	59	return ;\r
	60	}\r
	61	\r
	62	/**\r
	63	Allocate pages for creating 4KB-page based on 2MB-page when page fault happens.\r
	64	\r
	65	**/\r
	66	VOID\r
	67	InitPagesForPFHandler (\r
	68	VOID\r
	69	)\r
	70	{\r
	71	VOID *Address;\r
	72	\r
	73	//\r
	74	// Pre-Allocate memory for page fault handler\r
	75	//\r
	76	Address = NULL;\r
	77	Address = AllocatePages (MAX_PF_PAGE_COUNT);\r
ef3e20e3	78	ASSERT (Address != NULL);\r
427e3573 MK	79	\r
	80	mPFPageBuffer = (UINT64)(UINTN) Address;\r
	81	mPFPageIndex = 0;\r
	82	ZeroMem ((VOID ) (UINTN) mPFPageBuffer, EFI_PAGE_SIZE MAX_PF_PAGE_COUNT);\r
	83	ZeroMem (mPFPageUplink, sizeof (mPFPageUplink));\r
	84	\r
	85	return;\r
	86	}\r
	87	\r
	88	/**\r
	89	Allocate one page for creating 4KB-page based on 2MB-page.\r
	90	\r
	91	@param Uplink The address of Page-Directory entry.\r
	92	\r
	93	**/\r
	94	VOID\r
	95	AcquirePage (\r
	96	UINT64 *Uplink\r
	97	)\r
	98	{\r
	99	UINT64 Address;\r
	100	\r
	101	//\r
	102	// Get the buffer\r
	103	//\r
	104	Address = mPFPageBuffer + EFI_PAGES_TO_SIZE (mPFPageIndex);\r
	105	ZeroMem ((VOID *) (UINTN) Address, EFI_PAGE_SIZE);\r
	106	\r
	107	//\r
	108	// Cut the previous uplink if it exists and wasn't overwritten\r
	109	//\r
241f9149	110	if ((mPFPageUplink[mPFPageIndex] != NULL) && ((*mPFPageUplink[mPFPageIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK) == Address)) {\r
427e3573 MK	111	*mPFPageUplink[mPFPageIndex] = 0;\r
	112	}\r
	113	\r
	114	//\r
	115	// Link & Record the current uplink\r
	116	//\r
241f9149	117	*Uplink = Address \| mAddressEncMask \| PAGE_ATTRIBUTE_BITS;\r
427e3573 MK	118	mPFPageUplink[mPFPageIndex] = Uplink;\r
	119	\r
	120	mPFPageIndex = (mPFPageIndex + 1) % MAX_PF_PAGE_COUNT;\r
	121	}\r
	122	\r
	123	/**\r
	124	Update page table to map the memory correctly in order to make the instruction\r
	125	which caused page fault execute successfully. And it also save the original page\r
	126	table to be restored in single-step exception.\r
	127	\r
	128	@param PageTable PageTable Address.\r
	129	@param PFAddress The memory address which caused page fault exception.\r
	130	@param CpuIndex The index of the processor.\r
	131	@param ErrorCode The Error code of exception.\r
	132	@param IsValidPFAddress The flag indicates if SMM profile data need be added.\r
	133	\r
	134	**/\r
	135	VOID\r
	136	RestorePageTableAbove4G (\r
	137	UINT64 *PageTable,\r
	138	UINT64 PFAddress,\r
	139	UINTN CpuIndex,\r
	140	UINTN ErrorCode,\r
	141	BOOLEAN *IsValidPFAddress\r
	142	)\r
	143	{\r
	144	UINTN PTIndex;\r
	145	UINT64 Address;\r
	146	BOOLEAN Nx;\r
	147	BOOLEAN Existed;\r
	148	UINTN Index;\r
	149	UINTN PFIndex;\r
7365eb2c RN	150	IA32_CR4 Cr4;\r
7365eb2c RN	151	BOOLEAN Enable5LevelPaging;\r
427e3573 MK	152	\r
	153	ASSERT ((PageTable != NULL) && (IsValidPFAddress != NULL));\r
	154	\r
7365eb2c RN	155	Cr4.UintN = AsmReadCr4 ();\r
	156	Enable5LevelPaging = (BOOLEAN) (Cr4.Bits.LA57 == 1);\r
	157	\r
427e3573 MK	158	//\r
	159	// If page fault address is 4GB above.\r
	160	//\r
	161	\r
	162	//\r
	163	// Check if page fault address has existed in page table.\r
	164	// If it exists in page table but page fault is generated,\r
	165	// there are 2 possible reasons: 1. present flag is set to 0; 2. instruction fetch in protected memory range.\r
	166	//\r
	167	Existed = FALSE;\r
	168	PageTable = (UINT64*)(AsmReadCr3 () & PHYSICAL_ADDRESS_MASK);\r
7365eb2c RN	169	PTIndex = 0;\r
	170	if (Enable5LevelPaging) {\r
	171	PTIndex = BitFieldRead64 (PFAddress, 48, 56);\r
	172	}\r
	173	if ((!Enable5LevelPaging) \|\| ((PageTable[PTIndex] & IA32_PG_P) != 0)) {\r
	174	// PML5E\r
	175	if (Enable5LevelPaging) {\r
	176	PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
	177	}\r
	178	PTIndex = BitFieldRead64 (PFAddress, 39, 47);\r
427e3573	179	if ((PageTable[PTIndex] & IA32_PG_P) != 0) {\r
7365eb2c	180	// PML4E\r
241f9149	181	PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
7365eb2c RN	182	PTIndex = BitFieldRead64 (PFAddress, 30, 38);\r
	183	if ((PageTable[PTIndex] & IA32_PG_P) != 0) {\r
	184	// PDPTE\r
	185	PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
	186	PTIndex = BitFieldRead64 (PFAddress, 21, 29);\r
	187	// PD\r
	188	if ((PageTable[PTIndex] & IA32_PG_PS) != 0) {\r
427e3573	189	//\r
7365eb2c	190	// 2MB page\r
427e3573	191	//\r
241f9149	192	Address = (UINT64)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
7365eb2c	193	if ((Address & ~((1ull << 21) - 1)) == ((PFAddress & PHYSICAL_ADDRESS_MASK & ~((1ull << 21) - 1)))) {\r
427e3573 MK	194	Existed = TRUE;\r
427e3573 MK	195	}\r
7365eb2c RN	196	} else {\r
	197	//\r
	198	// 4KB page\r
	199	//\r
	200	PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask& PHYSICAL_ADDRESS_MASK);\r
	201	if (PageTable != 0) {\r
	202	//\r
	203	// When there is a valid entry to map to 4KB page, need not create a new entry to map 2MB.\r
	204	//\r
	205	PTIndex = BitFieldRead64 (PFAddress, 12, 20);\r
	206	Address = (UINT64)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
	207	if ((Address & ~((1ull << 12) - 1)) == (PFAddress & PHYSICAL_ADDRESS_MASK & ~((1ull << 12) - 1))) {\r
	208	Existed = TRUE;\r
	209	}\r
	210	}\r
427e3573 MK	211	}\r
	212	}\r
	213	}\r
	214	}\r
	215	\r
	216	//\r
	217	// If page entry does not existed in page table at all, create a new entry.\r
	218	//\r
	219	if (!Existed) {\r
	220	\r
	221	if (IsAddressValid (PFAddress, &Nx)) {\r
	222	//\r
	223	// If page fault address above 4GB is in protected range but it causes a page fault exception,\r
	224	// Will create a page entry for this page fault address, make page table entry as present/rw and execution-disable.\r
	225	// this access is not saved into SMM profile data.\r
	226	//\r
	227	*IsValidPFAddress = TRUE;\r
	228	}\r
	229	\r
	230	//\r
	231	// Create one entry in page table for page fault address.\r
	232	//\r
	233	SmiDefaultPFHandler ();\r
	234	//\r
	235	// Find the page table entry created just now.\r
	236	//\r
	237	PageTable = (UINT64*)(AsmReadCr3 () & PHYSICAL_ADDRESS_MASK);\r
	238	PFAddress = AsmReadCr2 ();\r
7365eb2c RN	239	// PML5E\r
	240	if (Enable5LevelPaging) {\r
	241	PTIndex = BitFieldRead64 (PFAddress, 48, 56);\r
	242	PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
	243	}\r
427e3573 MK	244	// PML4E\r
427e3573 MK	245	PTIndex = BitFieldRead64 (PFAddress, 39, 47);\r
241f9149	246	PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
427e3573 MK	247	// PDPTE\r
427e3573 MK	248	PTIndex = BitFieldRead64 (PFAddress, 30, 38);\r
241f9149	249	PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
427e3573 MK	250	// PD\r
427e3573 MK	251	PTIndex = BitFieldRead64 (PFAddress, 21, 29);\r
241f9149	252	Address = PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK;\r
427e3573 MK	253	//\r
	254	// Check if 2MB-page entry need be changed to 4KB-page entry.\r
	255	//\r
	256	if (IsAddressSplit (Address)) {\r
	257	AcquirePage (&PageTable[PTIndex]);\r
	258	\r
	259	// PTE\r
241f9149	260	PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
427e3573	261	for (Index = 0; Index < 512; Index++) {\r
241f9149	262	PageTable[Index] = Address \| mAddressEncMask \| PAGE_ATTRIBUTE_BITS;\r
427e3573	263	if (!IsAddressValid (Address, &Nx)) {\r
881520ea	264	PageTable[Index] = PageTable[Index] & (INTN)(INT32)(~PAGE_ATTRIBUTE_BITS);\r
427e3573 MK	265	}\r
	266	if (Nx && mXdSupported) {\r
	267	PageTable[Index] = PageTable[Index] \| IA32_PG_NX;\r
	268	}\r
	269	if (Address == (PFAddress & PHYSICAL_ADDRESS_MASK & ~((1ull << 12) - 1))) {\r
	270	PTIndex = Index;\r
	271	}\r
	272	Address += SIZE_4KB;\r
	273	} // end for PT\r
	274	} else {\r
	275	//\r
	276	// Update 2MB page entry.\r
	277	//\r
	278	if (!IsAddressValid (Address, &Nx)) {\r
	279	//\r
	280	// Patch to remove present flag and rw flag.\r
	281	//\r
881520ea	282	PageTable[PTIndex] = PageTable[PTIndex] & (INTN)(INT32)(~PAGE_ATTRIBUTE_BITS);\r
427e3573 MK	283	}\r
	284	//\r
	285	// Set XD bit to 1\r
	286	//\r
	287	if (Nx && mXdSupported) {\r
	288	PageTable[PTIndex] = PageTable[PTIndex] \| IA32_PG_NX;\r
	289	}\r
	290	}\r
	291	}\r
	292	\r
	293	//\r
	294	// Record old entries with non-present status\r
	295	// Old entries include the memory which instruction is at and the memory which instruction access.\r
	296	//\r
	297	//\r
	298	ASSERT (mPFEntryCount[CpuIndex] < MAX_PF_ENTRY_COUNT);\r
	299	if (mPFEntryCount[CpuIndex] < MAX_PF_ENTRY_COUNT) {\r
	300	PFIndex = mPFEntryCount[CpuIndex];\r
	301	mLastPFEntryValue[CpuIndex][PFIndex] = PageTable[PTIndex];\r
	302	mLastPFEntryPointer[CpuIndex][PFIndex] = &PageTable[PTIndex];\r
	303	mPFEntryCount[CpuIndex]++;\r
	304	}\r
	305	\r
	306	//\r
	307	// Add present flag or clear XD flag to make page fault handler succeed.\r
	308	//\r
881520ea	309	PageTable[PTIndex] \|= (UINT64)(PAGE_ATTRIBUTE_BITS);\r
427e3573 MK	310	if ((ErrorCode & IA32_PF_EC_ID) != 0) {\r
	311	//\r
	312	// If page fault is caused by instruction fetch, clear XD bit in the entry.\r
	313	//\r
	314	PageTable[PTIndex] &= ~IA32_PG_NX;\r
	315	}\r
	316	\r
	317	return;\r
	318	}\r
	319	\r
	320	/**\r
	321	Clear TF in FLAGS.\r
	322	\r
	323	@param SystemContext A pointer to the processor context when\r
	324	the interrupt occurred on the processor.\r
	325	\r
	326	**/\r
	327	VOID\r
	328	ClearTrapFlag (\r
	329	IN OUT EFI_SYSTEM_CONTEXT SystemContext\r
	330	)\r
	331	{\r
	332	SystemContext.SystemContextX64->Rflags &= (UINTN) ~BIT8;\r
	333	}\r