UefiCpuPkg/PiSmmCpu: Enable 5 level paging when CPU supports

REF:https://bugzilla.tianocore.org/show_bug.cgi?id=1946

The patch changes SMM environment to use 5 level paging when CPU
supports it.

Signed-off-by: Ray Ni <ray.ni@intel.com>
Cc: Eric Dong <eric.dong@intel.com>
Regression-tested-by: Laszlo Ersek <lersek@redhat.com>
Reviewed-by: Eric Dong <eric.dong@intel.com>

diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/SmmCpuMemoryManagement.c b/UefiCpuPkg/PiSmmCpuDxeSmm/SmmCpuMemoryManagement.c
index 069be3aaa5a5ed529593cf652b53e5f07092f1a2..55090e9c3e4fe79b32fb561763a781faa9015223 100644 (file)
--- a/UefiCpuPkg/PiSmmCpuDxeSmm/SmmCpuMemoryManagement.c
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/SmmCpuMemoryManagement.c
@@ -125,18 +125,36 @@ GetPageTableEntry (
   UINTN                 Index2;\r
   UINTN                 Index3;\r
   UINTN                 Index4;\r
+  UINTN                 Index5;\r
   UINT64                *L1PageTable;\r
   UINT64                *L2PageTable;\r
   UINT64                *L3PageTable;\r
   UINT64                *L4PageTable;\r
+  UINT64                *L5PageTable;\r
+  IA32_CR4              Cr4;\r
+  BOOLEAN               Enable5LevelPaging;\r
 \r
+  Index5 = ((UINTN)RShiftU64 (Address, 48)) & PAGING_PAE_INDEX_MASK;\r
   Index4 = ((UINTN)RShiftU64 (Address, 39)) & PAGING_PAE_INDEX_MASK;\r
   Index3 = ((UINTN)Address >> 30) & PAGING_PAE_INDEX_MASK;\r
   Index2 = ((UINTN)Address >> 21) & PAGING_PAE_INDEX_MASK;\r
   Index1 = ((UINTN)Address >> 12) & PAGING_PAE_INDEX_MASK;\r
 \r
+  Cr4.UintN = AsmReadCr4 ();\r
+  Enable5LevelPaging = (BOOLEAN) (Cr4.Bits.LA57 == 1);\r
+\r
   if (sizeof(UINTN) == sizeof(UINT64)) {\r
-    L4PageTable = (UINT64 *)GetPageTableBase ();\r
+    if (Enable5LevelPaging) {\r
+      L5PageTable = (UINT64 *)GetPageTableBase ();\r
+      if (L5PageTable[Index5] == 0) {\r
+        *PageAttribute = PageNone;\r
+        return NULL;\r
+      }\r
+\r
+      L4PageTable = (UINT64 *)(UINTN)(L5PageTable[Index5] & ~mAddressEncMask & PAGING_4K_ADDRESS_MASK_64);\r
+    } else {\r
+      L4PageTable = (UINT64 *)GetPageTableBase ();\r
+    }\r
     if (L4PageTable[Index4] == 0) {\r
       *PageAttribute = PageNone;\r
       return NULL;\r

diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/SmmProfile.c b/UefiCpuPkg/PiSmmCpuDxeSmm/SmmProfile.c
index e2b6a2d9b27ad8a77a9a2badeffdaf3bdb2a4406..c5131526f0c638555f922792938c0b0d02f04226 100644 (file)
--- a/UefiCpuPkg/PiSmmCpuDxeSmm/SmmProfile.c
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/SmmProfile.c
@@ -534,43 +534,78 @@ InitPaging (
   VOID\r
   )\r
 {\r
+  UINT64                            Pml5Entry;\r
+  UINT64                            Pml4Entry;\r
+  UINT64                            *Pml5;\r
   UINT64                            *Pml4;\r
   UINT64                            *Pdpt;\r
   UINT64                            *Pd;\r
   UINT64                            *Pt;\r
   UINTN                             Address;\r
+  UINTN                             Pml5Index;\r
   UINTN                             Pml4Index;\r
   UINTN                             PdptIndex;\r
   UINTN                             PdIndex;\r
   UINTN                             PtIndex;\r
   UINTN                             NumberOfPdptEntries;\r
   UINTN                             NumberOfPml4Entries;\r
+  UINTN                             NumberOfPml5Entries;\r
   UINTN                             SizeOfMemorySpace;\r
   BOOLEAN                           Nx;\r
+  IA32_CR4                          Cr4;\r
+  BOOLEAN                           Enable5LevelPaging;\r
+\r
+  Cr4.UintN = AsmReadCr4 ();\r
+  Enable5LevelPaging = (BOOLEAN) (Cr4.Bits.LA57 == 1);\r
 \r
   if (sizeof (UINTN) == sizeof (UINT64)) {\r
-    Pml4 = (UINT64*)(UINTN)mSmmProfileCr3;\r
+    if (!Enable5LevelPaging) {\r
+      Pml5Entry = (UINTN) mSmmProfileCr3 | IA32_PG_P;\r
+      Pml5 = &Pml5Entry;\r
+    } else {\r
+      Pml5 = (UINT64*) (UINTN) mSmmProfileCr3;\r
+    }\r
     SizeOfMemorySpace = HighBitSet64 (gPhyMask) + 1;\r
     //\r
     // Calculate the table entries of PML4E and PDPTE.\r
     //\r
-    if (SizeOfMemorySpace <= 39 ) {\r
-      NumberOfPml4Entries = 1;\r
-      NumberOfPdptEntries = (UINT32)LShiftU64 (1, (SizeOfMemorySpace - 30));\r
-    } else {\r
-      NumberOfPml4Entries = (UINT32)LShiftU64 (1, (SizeOfMemorySpace - 39));\r
-      NumberOfPdptEntries = 512;\r
+    NumberOfPml5Entries = 1;\r
+    if (SizeOfMemorySpace > 48) {\r
+      NumberOfPml5Entries = (UINTN) LShiftU64 (1, SizeOfMemorySpace - 48);\r
+      SizeOfMemorySpace = 48;\r
     }\r
-  } else {\r
+\r
     NumberOfPml4Entries = 1;\r
+    if (SizeOfMemorySpace > 39) {\r
+      NumberOfPml4Entries = (UINTN) LShiftU64 (1, SizeOfMemorySpace - 39);\r
+      SizeOfMemorySpace = 39;\r
+    }\r
+\r
+    NumberOfPdptEntries = 1;\r
+    ASSERT (SizeOfMemorySpace > 30);\r
+    NumberOfPdptEntries = (UINTN) LShiftU64 (1, SizeOfMemorySpace - 30);\r
+  } else {\r
+    Pml4Entry = (UINTN) mSmmProfileCr3 | IA32_PG_P;\r
+    Pml4 = &Pml4Entry;\r
+    Pml5Entry = (UINTN) Pml4 | IA32_PG_P;\r
+    Pml5 = &Pml5Entry;\r
+    NumberOfPml5Entries  = 1;\r
+    NumberOfPml4Entries  = 1;\r
     NumberOfPdptEntries  = 4;\r
   }\r
 \r
   //\r
   // Go through page table and change 2MB-page into 4KB-page.\r
   //\r
-  for (Pml4Index = 0; Pml4Index < NumberOfPml4Entries; Pml4Index++) {\r
-    if (sizeof (UINTN) == sizeof (UINT64)) {\r
+  for (Pml5Index = 0; Pml5Index < NumberOfPml5Entries; Pml5Index++) {\r
+    if ((Pml5[Pml5Index] & IA32_PG_P) == 0) {\r
+      //\r
+      // If PML5 entry does not exist, skip it\r
+      //\r
+      continue;\r
+    }\r
+    Pml4 = (UINT64 *) (UINTN) (Pml5[Pml5Index] & PHYSICAL_ADDRESS_MASK);\r
+    for (Pml4Index = 0; Pml4Index < NumberOfPml4Entries; Pml4Index++) {\r
       if ((Pml4[Pml4Index] & IA32_PG_P) == 0) {\r
         //\r
         // If PML4 entry does not exist, skip it\r
@@ -578,63 +613,76 @@ InitPaging (
         continue;\r
       }\r
       Pdpt = (UINT64 *)(UINTN)(Pml4[Pml4Index] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
-    } else {\r
-      Pdpt = (UINT64*)(UINTN)mSmmProfileCr3;\r
-    }\r
-    for (PdptIndex = 0; PdptIndex < NumberOfPdptEntries; PdptIndex++, Pdpt++) {\r
-      if ((*Pdpt & IA32_PG_P) == 0) {\r
-        //\r
-        // If PDPT entry does not exist, skip it\r
-        //\r
-        continue;\r
-      }\r
-      if ((*Pdpt & IA32_PG_PS) != 0) {\r
-        //\r
-        // This is 1G entry, skip it\r
-        //\r
-        continue;\r
-      }\r
-      Pd = (UINT64 *)(UINTN)(*Pdpt & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
-      if (Pd == 0) {\r
-        continue;\r
-      }\r
-      for (PdIndex = 0; PdIndex < SIZE_4KB / sizeof (*Pd); PdIndex++, Pd++) {\r
-        if ((*Pd & IA32_PG_P) == 0) {\r
+      for (PdptIndex = 0; PdptIndex < NumberOfPdptEntries; PdptIndex++, Pdpt++) {\r
+        if ((*Pdpt & IA32_PG_P) == 0) {\r
+          //\r
+          // If PDPT entry does not exist, skip it\r
+          //\r
+          continue;\r
+        }\r
+        if ((*Pdpt & IA32_PG_PS) != 0) {\r
           //\r
-          // If PD entry does not exist, skip it\r
+          // This is 1G entry, skip it\r
           //\r
           continue;\r
         }\r
-        Address = (((PdptIndex << 9) + PdIndex) << 21);\r
+        Pd = (UINT64 *)(UINTN)(*Pdpt & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
+        if (Pd == 0) {\r
+          continue;\r
+        }\r
+        for (PdIndex = 0; PdIndex < SIZE_4KB / sizeof (*Pd); PdIndex++, Pd++) {\r
+          if ((*Pd & IA32_PG_P) == 0) {\r
+            //\r
+            // If PD entry does not exist, skip it\r
+            //\r
+            continue;\r
+          }\r
+          Address = (UINTN) LShiftU64 (\r
+                              LShiftU64 (\r
+                                LShiftU64 ((Pml5Index << 9) + Pml4Index, 9) + PdptIndex,\r
+                                9\r
+                                ) + PdIndex,\r
+                                21\r
+                              );\r
 \r
-        //\r
-        // If it is 2M page, check IsAddressSplit()\r
-        //\r
-        if (((*Pd & IA32_PG_PS) != 0) && IsAddressSplit (Address)) {\r
           //\r
-          // Based on current page table, create 4KB page table for split area.\r
+          // If it is 2M page, check IsAddressSplit()\r
           //\r
-          ASSERT (Address == (*Pd & PHYSICAL_ADDRESS_MASK));\r
+          if (((*Pd & IA32_PG_PS) != 0) && IsAddressSplit (Address)) {\r
+            //\r
+            // Based on current page table, create 4KB page table for split area.\r
+            //\r
+            ASSERT (Address == (*Pd & PHYSICAL_ADDRESS_MASK));\r
+\r
+            Pt = AllocatePageTableMemory (1);\r
+            ASSERT (Pt != NULL);\r
 \r
-          Pt = AllocatePageTableMemory (1);\r
-          ASSERT (Pt != NULL);\r
+            *Pd = (UINTN) Pt | IA32_PG_RW | IA32_PG_P;\r
 \r
-          // Split it\r
-          for (PtIndex = 0; PtIndex < SIZE_4KB / sizeof(*Pt); PtIndex++) {\r
-            Pt[PtIndex] = Address + ((PtIndex << 12) | mAddressEncMask | PAGE_ATTRIBUTE_BITS);\r
-          } // end for PT\r
-          *Pd = (UINT64)(UINTN)Pt | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
-        } // end if IsAddressSplit\r
-      } // end for PD\r
-    } // end for PDPT\r
-  } // end for PML4\r
+            // Split it\r
+            for (PtIndex = 0; PtIndex < SIZE_4KB / sizeof(*Pt); PtIndex++, Pt++) {\r
+              *Pt = Address + ((PtIndex << 12) | mAddressEncMask | PAGE_ATTRIBUTE_BITS);\r
+            } // end for PT\r
+            *Pd = (UINT64)(UINTN)Pt | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
+          } // end if IsAddressSplit\r
+        } // end for PD\r
+      } // end for PDPT\r
+    } // end for PML4\r
+  } // end for PML5\r
 \r
   //\r
   // Go through page table and set several page table entries to absent or execute-disable.\r
   //\r
   DEBUG ((EFI_D_INFO, "Patch page table start ...\n"));\r
-  for (Pml4Index = 0; Pml4Index < NumberOfPml4Entries; Pml4Index++) {\r
-    if (sizeof (UINTN) == sizeof (UINT64)) {\r
+  for (Pml5Index = 0; Pml5Index < NumberOfPml5Entries; Pml5Index++) {\r
+    if ((Pml5[Pml5Index] & IA32_PG_P) == 0) {\r
+      //\r
+      // If PML5 entry does not exist, skip it\r
+      //\r
+      continue;\r
+    }\r
+    Pml4 = (UINT64 *) (UINTN) (Pml5[Pml5Index] & PHYSICAL_ADDRESS_MASK);\r
+    for (Pml4Index = 0; Pml4Index < NumberOfPml4Entries; Pml4Index++) {\r
       if ((Pml4[Pml4Index] & IA32_PG_P) == 0) {\r
         //\r
         // If PML4 entry does not exist, skip it\r
@@ -642,69 +690,73 @@ InitPaging (
         continue;\r
       }\r
       Pdpt = (UINT64 *)(UINTN)(Pml4[Pml4Index] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
-    } else {\r
-      Pdpt = (UINT64*)(UINTN)mSmmProfileCr3;\r
-    }\r
-    for (PdptIndex = 0; PdptIndex < NumberOfPdptEntries; PdptIndex++, Pdpt++) {\r
-      if ((*Pdpt & IA32_PG_P) == 0) {\r
-        //\r
-        // If PDPT entry does not exist, skip it\r
-        //\r
-        continue;\r
-      }\r
-      if ((*Pdpt & IA32_PG_PS) != 0) {\r
-        //\r
-        // This is 1G entry, set NX bit and skip it\r
-        //\r
-        if (mXdSupported) {\r
-          *Pdpt = *Pdpt | IA32_PG_NX;\r
+      for (PdptIndex = 0; PdptIndex < NumberOfPdptEntries; PdptIndex++, Pdpt++) {\r
+        if ((*Pdpt & IA32_PG_P) == 0) {\r
+          //\r
+          // If PDPT entry does not exist, skip it\r
+          //\r
+          continue;\r
         }\r
-        continue;\r
-      }\r
-      Pd = (UINT64 *)(UINTN)(*Pdpt & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
-      if (Pd == 0) {\r
-        continue;\r
-      }\r
-      for (PdIndex = 0; PdIndex < SIZE_4KB / sizeof (*Pd); PdIndex++, Pd++) {\r
-        if ((*Pd & IA32_PG_P) == 0) {\r
+        if ((*Pdpt & IA32_PG_PS) != 0) {\r
           //\r
-          // If PD entry does not exist, skip it\r
+          // This is 1G entry, set NX bit and skip it\r
           //\r
+          if (mXdSupported) {\r
+            *Pdpt = *Pdpt | IA32_PG_NX;\r
+          }\r
           continue;\r
         }\r
-        Address = (((PdptIndex << 9) + PdIndex) << 21);\r
-\r
-        if ((*Pd & IA32_PG_PS) != 0) {\r
-          // 2MB page\r
-\r
-          if (!IsAddressValid (Address, &Nx)) {\r
+        Pd = (UINT64 *)(UINTN)(*Pdpt & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
+        if (Pd == 0) {\r
+          continue;\r
+        }\r
+        for (PdIndex = 0; PdIndex < SIZE_4KB / sizeof (*Pd); PdIndex++, Pd++) {\r
+          if ((*Pd & IA32_PG_P) == 0) {\r
             //\r
-            // Patch to remove Present flag and RW flag\r
+            // If PD entry does not exist, skip it\r
             //\r
-            *Pd = *Pd & (INTN)(INT32)(~PAGE_ATTRIBUTE_BITS);\r
-          }\r
-          if (Nx && mXdSupported) {\r
-            *Pd = *Pd | IA32_PG_NX;\r
-          }\r
-        } else {\r
-          // 4KB page\r
-          Pt = (UINT64 *)(UINTN)(*Pd & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
-          if (Pt == 0) {\r
             continue;\r
           }\r
-          for (PtIndex = 0; PtIndex < SIZE_4KB / sizeof(*Pt); PtIndex++, Pt++) {\r
+          Address = (UINTN) LShiftU64 (\r
+                              LShiftU64 (\r
+                                LShiftU64 ((Pml5Index << 9) + Pml4Index, 9) + PdptIndex,\r
+                                9\r
+                                ) + PdIndex,\r
+                                21\r
+                              );\r
+\r
+          if ((*Pd & IA32_PG_PS) != 0) {\r
+            // 2MB page\r
+\r
             if (!IsAddressValid (Address, &Nx)) {\r
-              *Pt = *Pt & (INTN)(INT32)(~PAGE_ATTRIBUTE_BITS);\r
+              //\r
+              // Patch to remove Present flag and RW flag\r
+              //\r
+              *Pd = *Pd & (INTN)(INT32)(~PAGE_ATTRIBUTE_BITS);\r
             }\r
             if (Nx && mXdSupported) {\r
-              *Pt = *Pt | IA32_PG_NX;\r
+              *Pd = *Pd | IA32_PG_NX;\r
+            }\r
+          } else {\r
+            // 4KB page\r
+            Pt = (UINT64 *)(UINTN)(*Pd & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
+            if (Pt == 0) {\r
+              continue;\r
             }\r
-            Address += SIZE_4KB;\r
-          } // end for PT\r
-        } // end if PS\r
-      } // end for PD\r
-    } // end for PDPT\r
-  } // end for PML4\r
+            for (PtIndex = 0; PtIndex < SIZE_4KB / sizeof(*Pt); PtIndex++, Pt++) {\r
+              if (!IsAddressValid (Address, &Nx)) {\r
+                *Pt = *Pt & (INTN)(INT32)(~PAGE_ATTRIBUTE_BITS);\r
+              }\r
+              if (Nx && mXdSupported) {\r
+                *Pt = *Pt | IA32_PG_NX;\r
+              }\r
+              Address += SIZE_4KB;\r
+            } // end for PT\r
+          } // end if PS\r
+        } // end for PD\r
+      } // end for PDPT\r
+    } // end for PML4\r
+  } // end for PML5\r
 \r
   //\r
   // Flush TLB\r
@@ -1156,6 +1208,20 @@ RestorePageTableBelow4G (
 {\r
   UINTN         PTIndex;\r
   UINTN         PFIndex;\r
+  IA32_CR4      Cr4;\r
+  BOOLEAN       Enable5LevelPaging;\r
+\r
+  Cr4.UintN = AsmReadCr4 ();\r
+  Enable5LevelPaging = (BOOLEAN) (Cr4.Bits.LA57 == 1);\r
+\r
+  //\r
+  // PML5\r
+  //\r
+  if (Enable5LevelPaging) {\r
+    PTIndex = (UINTN)BitFieldRead64 (PFAddress, 48, 56);\r
+    ASSERT (PageTable[PTIndex] != 0);\r
+    PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & PHYSICAL_ADDRESS_MASK);\r
+  }\r
 \r
   //\r
   // PML4\r

diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/X64/PageTbl.c b/UefiCpuPkg/PiSmmCpuDxeSmm/X64/PageTbl.c
index 3d5d663d990095547b6dae9e3b0cabe44929df77..c31160735a3790530058026df0a8009cc4be0d6d 100644 (file)
--- a/UefiCpuPkg/PiSmmCpuDxeSmm/X64/PageTbl.c
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/X64/PageTbl.c
@@ -16,6 +16,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent
 LIST_ENTRY                          mPagePool = INITIALIZE_LIST_HEAD_VARIABLE (mPagePool);\r
 BOOLEAN                             m1GPageTableSupport = FALSE;\r
 BOOLEAN                             mCpuSmmStaticPageTable;\r
+BOOLEAN                             m5LevelPagingSupport;\r
+X86_ASSEMBLY_PATCH_LABEL            gPatch5LevelPagingSupport;\r
 \r
 /**\r
   Disable CET.\r
@@ -60,6 +62,31 @@ Is1GPageSupport (
   return FALSE;\r
 }\r
 \r
+/**\r
+  Check if 5-level paging is supported by processor or not.\r
+\r
+  @retval TRUE   5-level paging is supported.\r
+  @retval FALSE  5-level paging is not supported.\r
+\r
+**/\r
+BOOLEAN\r
+Is5LevelPagingSupport (\r
+  VOID\r
+  )\r
+{\r
+  CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS_ECX EcxFlags;\r
+\r
+  AsmCpuidEx (\r
+    CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS,\r
+    CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS_SUB_LEAF_INFO,\r
+    NULL,\r
+    NULL,\r
+    &EcxFlags.Uint32,\r
+    NULL\r
+    );\r
+  return (BOOLEAN) (EcxFlags.Bits.FiveLevelPage != 0);\r
+}\r
+\r
 /**\r
   Set sub-entries number in entry.\r
 \r
@@ -130,14 +157,6 @@ CalculateMaximumSupportAddress (
       PhysicalAddressBits = 36;\r
     }\r
   }\r
-\r
-  //\r
-  // IA-32e paging translates 48-bit linear addresses to 52-bit physical addresses.\r
-  //\r
-  ASSERT (PhysicalAddressBits <= 52);\r
-  if (PhysicalAddressBits > 48) {\r
-    PhysicalAddressBits = 48;\r
-  }\r
   return PhysicalAddressBits;\r
 }\r
 \r
@@ -152,89 +171,137 @@ SetStaticPageTable (
   )\r
 {\r
   UINT64                                        PageAddress;\r
+  UINTN                                         NumberOfPml5EntriesNeeded;\r
   UINTN                                         NumberOfPml4EntriesNeeded;\r
   UINTN                                         NumberOfPdpEntriesNeeded;\r
+  UINTN                                         IndexOfPml5Entries;\r
   UINTN                                         IndexOfPml4Entries;\r
   UINTN                                         IndexOfPdpEntries;\r
   UINTN                                         IndexOfPageDirectoryEntries;\r
+  UINT64                                        *PageMapLevel5Entry;\r
   UINT64                                        *PageMapLevel4Entry;\r
   UINT64                                        *PageMap;\r
   UINT64                                        *PageDirectoryPointerEntry;\r
   UINT64                                        *PageDirectory1GEntry;\r
   UINT64                                        *PageDirectoryEntry;\r
 \r
-  if (mPhysicalAddressBits <= 39 ) {\r
-    NumberOfPml4EntriesNeeded = 1;\r
-    NumberOfPdpEntriesNeeded = (UINT32)LShiftU64 (1, (mPhysicalAddressBits - 30));\r
-  } else {\r
-    NumberOfPml4EntriesNeeded = (UINT32)LShiftU64 (1, (mPhysicalAddressBits - 39));\r
-    NumberOfPdpEntriesNeeded = 512;\r
+  //\r
+  // IA-32e paging translates 48-bit linear addresses to 52-bit physical addresses\r
+  //  when 5-Level Paging is disabled.\r
+  //\r
+  ASSERT (mPhysicalAddressBits <= 52);\r
+  if (!m5LevelPagingSupport && mPhysicalAddressBits > 48) {\r
+    mPhysicalAddressBits = 48;\r
+  }\r
+\r
+  NumberOfPml5EntriesNeeded = 1;\r
+  if (mPhysicalAddressBits > 48) {\r
+    NumberOfPml5EntriesNeeded = (UINTN) LShiftU64 (1, mPhysicalAddressBits - 48);\r
+    mPhysicalAddressBits = 48;\r
+  }\r
+\r
+  NumberOfPml4EntriesNeeded = 1;\r
+  if (mPhysicalAddressBits > 39) {\r
+    NumberOfPml4EntriesNeeded = (UINTN) LShiftU64 (1, mPhysicalAddressBits - 39);\r
+    mPhysicalAddressBits = 39;\r
   }\r
 \r
+  NumberOfPdpEntriesNeeded = 1;\r
+  ASSERT (mPhysicalAddressBits > 30);\r
+  NumberOfPdpEntriesNeeded = (UINTN) LShiftU64 (1, mPhysicalAddressBits - 30);\r
+\r
   //\r
   // By architecture only one PageMapLevel4 exists - so lets allocate storage for it.\r
   //\r
   PageMap         = (VOID *) PageTable;\r
 \r
   PageMapLevel4Entry = PageMap;\r
-  PageAddress        = 0;\r
-  for (IndexOfPml4Entries = 0; IndexOfPml4Entries < NumberOfPml4EntriesNeeded; IndexOfPml4Entries++, PageMapLevel4Entry++) {\r
+  PageMapLevel5Entry = NULL;\r
+  if (m5LevelPagingSupport) {\r
     //\r
-    // Each PML4 entry points to a page of Page Directory Pointer entries.\r
+    // By architecture only one PageMapLevel5 exists - so lets allocate storage for it.\r
     //\r
-    PageDirectoryPointerEntry = (UINT64 *) ((*PageMapLevel4Entry) & ~mAddressEncMask & gPhyMask);\r
-    if (PageDirectoryPointerEntry == NULL) {\r
-      PageDirectoryPointerEntry = AllocatePageTableMemory (1);\r
-      ASSERT(PageDirectoryPointerEntry != NULL);\r
-      ZeroMem (PageDirectoryPointerEntry, EFI_PAGES_TO_SIZE(1));\r
+    PageMapLevel5Entry = PageMap;\r
+  }\r
+  PageAddress        = 0;\r
 \r
-      *PageMapLevel4Entry = (UINT64)(UINTN)PageDirectoryPointerEntry | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
+  for ( IndexOfPml5Entries = 0\r
+      ; IndexOfPml5Entries < NumberOfPml5EntriesNeeded\r
+      ; IndexOfPml5Entries++, PageMapLevel5Entry++) {\r
+    //\r
+    // Each PML5 entry points to a page of PML4 entires.\r
+    // So lets allocate space for them and fill them in in the IndexOfPml4Entries loop.\r
+    // When 5-Level Paging is disabled, below allocation happens only once.\r
+    //\r
+    if (m5LevelPagingSupport) {\r
+      PageMapLevel4Entry = (UINT64 *) ((*PageMapLevel5Entry) & ~mAddressEncMask & gPhyMask);\r
+      if (PageMapLevel4Entry == NULL) {\r
+        PageMapLevel4Entry = AllocatePageTableMemory (1);\r
+        ASSERT(PageMapLevel4Entry != NULL);\r
+        ZeroMem (PageMapLevel4Entry, EFI_PAGES_TO_SIZE(1));\r
+\r
+        *PageMapLevel5Entry = (UINT64)(UINTN)PageMapLevel4Entry | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
+      }\r
     }\r
 \r
-    if (m1GPageTableSupport) {\r
-      PageDirectory1GEntry = PageDirectoryPointerEntry;\r
-      for (IndexOfPageDirectoryEntries = 0; IndexOfPageDirectoryEntries < 512; IndexOfPageDirectoryEntries++, PageDirectory1GEntry++, PageAddress += SIZE_1GB) {\r
-        if (IndexOfPml4Entries == 0 && IndexOfPageDirectoryEntries < 4) {\r
-          //\r
-          // Skip the < 4G entries\r
-          //\r
-          continue;\r
-        }\r
-        //\r
-        // Fill in the Page Directory entries\r
-        //\r
-        *PageDirectory1GEntry = PageAddress | mAddressEncMask | IA32_PG_PS | PAGE_ATTRIBUTE_BITS;\r
+    for (IndexOfPml4Entries = 0; IndexOfPml4Entries < (NumberOfPml5EntriesNeeded == 1 ? NumberOfPml4EntriesNeeded : 512); IndexOfPml4Entries++, PageMapLevel4Entry++) {\r
+      //\r
+      // Each PML4 entry points to a page of Page Directory Pointer entries.\r
+      //\r
+      PageDirectoryPointerEntry = (UINT64 *) ((*PageMapLevel4Entry) & ~mAddressEncMask & gPhyMask);\r
+      if (PageDirectoryPointerEntry == NULL) {\r
+        PageDirectoryPointerEntry = AllocatePageTableMemory (1);\r
+        ASSERT(PageDirectoryPointerEntry != NULL);\r
+        ZeroMem (PageDirectoryPointerEntry, EFI_PAGES_TO_SIZE(1));\r
+\r
+        *PageMapLevel4Entry = (UINT64)(UINTN)PageDirectoryPointerEntry | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
       }\r
-    } else {\r
-      PageAddress = BASE_4GB;\r
-      for (IndexOfPdpEntries = 0; IndexOfPdpEntries < NumberOfPdpEntriesNeeded; IndexOfPdpEntries++, PageDirectoryPointerEntry++) {\r
-        if (IndexOfPml4Entries == 0 && IndexOfPdpEntries < 4) {\r
-          //\r
-          // Skip the < 4G entries\r
-          //\r
-          continue;\r
-        }\r
-        //\r
-        // Each Directory Pointer entries points to a page of Page Directory entires.\r
-        // So allocate space for them and fill them in in the IndexOfPageDirectoryEntries loop.\r
-        //\r
-        PageDirectoryEntry = (UINT64 *) ((*PageDirectoryPointerEntry) & ~mAddressEncMask & gPhyMask);\r
-        if (PageDirectoryEntry == NULL) {\r
-          PageDirectoryEntry = AllocatePageTableMemory (1);\r
-          ASSERT(PageDirectoryEntry != NULL);\r
-          ZeroMem (PageDirectoryEntry, EFI_PAGES_TO_SIZE(1));\r
 \r
+      if (m1GPageTableSupport) {\r
+        PageDirectory1GEntry = PageDirectoryPointerEntry;\r
+        for (IndexOfPageDirectoryEntries = 0; IndexOfPageDirectoryEntries < 512; IndexOfPageDirectoryEntries++, PageDirectory1GEntry++, PageAddress += SIZE_1GB) {\r
+          if (IndexOfPml4Entries == 0 && IndexOfPageDirectoryEntries < 4) {\r
+            //\r
+            // Skip the < 4G entries\r
+            //\r
+            continue;\r
+          }\r
           //\r
-          // Fill in a Page Directory Pointer Entries\r
+          // Fill in the Page Directory entries\r
           //\r
-          *PageDirectoryPointerEntry = (UINT64)(UINTN)PageDirectoryEntry | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
+          *PageDirectory1GEntry = PageAddress | mAddressEncMask | IA32_PG_PS | PAGE_ATTRIBUTE_BITS;\r
         }\r
-\r
-        for (IndexOfPageDirectoryEntries = 0; IndexOfPageDirectoryEntries < 512; IndexOfPageDirectoryEntries++, PageDirectoryEntry++, PageAddress += SIZE_2MB) {\r
+      } else {\r
+        PageAddress = BASE_4GB;\r
+        for (IndexOfPdpEntries = 0; IndexOfPdpEntries < (NumberOfPml4EntriesNeeded == 1 ? NumberOfPdpEntriesNeeded : 512); IndexOfPdpEntries++, PageDirectoryPointerEntry++) {\r
+          if (IndexOfPml4Entries == 0 && IndexOfPdpEntries < 4) {\r
+            //\r
+            // Skip the < 4G entries\r
+            //\r
+            continue;\r
+          }\r
           //\r
-          // Fill in the Page Directory entries\r
+          // Each Directory Pointer entries points to a page of Page Directory entires.\r
+          // So allocate space for them and fill them in in the IndexOfPageDirectoryEntries loop.\r
           //\r
-          *PageDirectoryEntry = PageAddress | mAddressEncMask | IA32_PG_PS | PAGE_ATTRIBUTE_BITS;\r
+          PageDirectoryEntry = (UINT64 *) ((*PageDirectoryPointerEntry) & ~mAddressEncMask & gPhyMask);\r
+          if (PageDirectoryEntry == NULL) {\r
+            PageDirectoryEntry = AllocatePageTableMemory (1);\r
+            ASSERT(PageDirectoryEntry != NULL);\r
+            ZeroMem (PageDirectoryEntry, EFI_PAGES_TO_SIZE(1));\r
+\r
+            //\r
+            // Fill in a Page Directory Pointer Entries\r
+            //\r
+            *PageDirectoryPointerEntry = (UINT64)(UINTN)PageDirectoryEntry | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
+          }\r
+\r
+          for (IndexOfPageDirectoryEntries = 0; IndexOfPageDirectoryEntries < 512; IndexOfPageDirectoryEntries++, PageDirectoryEntry++, PageAddress += SIZE_2MB) {\r
+            //\r
+            // Fill in the Page Directory entries\r
+            //\r
+            *PageDirectoryEntry = PageAddress | mAddressEncMask | IA32_PG_PS | PAGE_ATTRIBUTE_BITS;\r
+          }\r
         }\r
       }\r
     }\r
@@ -259,6 +326,8 @@ SmmInitPageTable (
   UINTN                             PageFaultHandlerHookAddress;\r
   IA32_IDT_GATE_DESCRIPTOR          *IdtEntry;\r
   EFI_STATUS                        Status;\r
+  UINT64                            *Pml4Entry;\r
+  UINT64                            *Pml5Entry;\r
 \r
   //\r
   // Initialize spin lock\r
@@ -266,12 +335,14 @@ SmmInitPageTable (
   InitializeSpinLock (mPFLock);\r
 \r
   mCpuSmmStaticPageTable = PcdGetBool (PcdCpuSmmStaticPageTable);\r
-  m1GPageTableSupport = Is1GPageSupport ();\r
-  DEBUG ((DEBUG_INFO, "1GPageTableSupport - 0x%x\n", m1GPageTableSupport));\r
-  DEBUG ((DEBUG_INFO, "PcdCpuSmmStaticPageTable - 0x%x\n", mCpuSmmStaticPageTable));\r
-\r
-  mPhysicalAddressBits = CalculateMaximumSupportAddress ();\r
-  DEBUG ((DEBUG_INFO, "PhysicalAddressBits - 0x%x\n", mPhysicalAddressBits));\r
+  m1GPageTableSupport    = Is1GPageSupport ();\r
+  m5LevelPagingSupport   = Is5LevelPagingSupport ();\r
+  mPhysicalAddressBits   = CalculateMaximumSupportAddress ();\r
+  PatchInstructionX86 (gPatch5LevelPagingSupport, m5LevelPagingSupport, 1);\r
+  DEBUG ((DEBUG_INFO, "5LevelPaging Support     - %d\n", m5LevelPagingSupport));\r
+  DEBUG ((DEBUG_INFO, "1GPageTable Support      - %d\n", m1GPageTableSupport));\r
+  DEBUG ((DEBUG_INFO, "PcdCpuSmmStaticPageTable - %d\n", mCpuSmmStaticPageTable));\r
+  DEBUG ((DEBUG_INFO, "PhysicalAddressBits      - %d\n", mPhysicalAddressBits));\r
   //\r
   // Generate PAE page table for the first 4GB memory space\r
   //\r
@@ -288,15 +359,30 @@ SmmInitPageTable (
   //\r
   // Fill Page-Table-Level4 (PML4) entry\r
   //\r
-  PTEntry = (UINT64*)AllocatePageTableMemory (1);\r
-  ASSERT (PTEntry != NULL);\r
-  *PTEntry = Pages | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
-  ZeroMem (PTEntry + 1, EFI_PAGE_SIZE - sizeof (*PTEntry));\r
+  Pml4Entry = (UINT64*)AllocatePageTableMemory (1);\r
+  ASSERT (Pml4Entry != NULL);\r
+  *Pml4Entry = Pages | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
+  ZeroMem (Pml4Entry + 1, EFI_PAGE_SIZE - sizeof (*Pml4Entry));\r
 \r
   //\r
   // Set sub-entries number\r
   //\r
-  SetSubEntriesNum (PTEntry, 3);\r
+  SetSubEntriesNum (Pml4Entry, 3);\r
+  PTEntry = Pml4Entry;\r
+\r
+  if (m5LevelPagingSupport) {\r
+    //\r
+    // Fill PML5 entry\r
+    //\r
+    Pml5Entry = (UINT64*)AllocatePageTableMemory (1);\r
+    *Pml5Entry = (UINTN) Pml4Entry | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
+    ZeroMem (Pml5Entry + 1, EFI_PAGE_SIZE - sizeof (*Pml5Entry));\r
+    //\r
+    // Set sub-entries number\r
+    //\r
+    SetSubEntriesNum (Pml5Entry, 1);\r
+    PTEntry = Pml5Entry;\r
+  }\r
 \r
   if (mCpuSmmStaticPageTable) {\r
     SetStaticPageTable ((UINTN)PTEntry);\r
@@ -344,7 +430,7 @@ SmmInitPageTable (
   }\r
 \r
   //\r
-  // Return the address of PML4 (to set CR3)\r
+  // Return the address of PML4/PML5 (to set CR3)\r
   //\r
   return (UINT32)(UINTN)PTEntry;\r
 }\r
@@ -436,12 +522,16 @@ ReclaimPages (
   VOID\r
   )\r
 {\r
+  UINT64                       Pml5Entry;\r
+  UINT64                       *Pml5;\r
   UINT64                       *Pml4;\r
   UINT64                       *Pdpt;\r
   UINT64                       *Pdt;\r
+  UINTN                        Pml5Index;\r
   UINTN                        Pml4Index;\r
   UINTN                        PdptIndex;\r
   UINTN                        PdtIndex;\r
+  UINTN                        MinPml5;\r
   UINTN                        MinPml4;\r
   UINTN                        MinPdpt;\r
   UINTN                        MinPdt;\r
@@ -451,120 +541,147 @@ ReclaimPages (
   BOOLEAN                      PML4EIgnore;\r
   BOOLEAN                      PDPTEIgnore;\r
   UINT64                       *ReleasePageAddress;\r
+  IA32_CR4                     Cr4;\r
+  BOOLEAN                      Enable5LevelPaging;\r
 \r
   Pml4 = NULL;\r
   Pdpt = NULL;\r
   Pdt  = NULL;\r
   MinAcc  = (UINT64)-1;\r
   MinPml4 = (UINTN)-1;\r
+  MinPml5 = (UINTN)-1;\r
   MinPdpt = (UINTN)-1;\r
   MinPdt  = (UINTN)-1;\r
   Acc     = 0;\r
   ReleasePageAddress = 0;\r
 \r
+  Cr4.UintN = AsmReadCr4 ();\r
+  Enable5LevelPaging = (BOOLEAN) (Cr4.Bits.LA57 == 1);\r
+  Pml5 = (UINT64*)(UINTN)(AsmReadCr3 () & gPhyMask);\r
+\r
+  if (!Enable5LevelPaging) {\r
+    //\r
+    // Create one fake PML5 entry for 4-Level Paging\r
+    // so that the page table parsing logic only handles 5-Level page structure.\r
+    //\r
+    Pml5Entry = (UINTN) Pml5 | IA32_PG_P;\r
+    Pml5 = &Pml5Entry;\r
+  }\r
+\r
   //\r
   // First, find the leaf entry has the smallest access record value\r
   //\r
-  Pml4 = (UINT64*)(UINTN)(AsmReadCr3 () & gPhyMask);\r
-  for (Pml4Index = 0; Pml4Index < EFI_PAGE_SIZE / sizeof (*Pml4); Pml4Index++) {\r
-    if ((Pml4[Pml4Index] & IA32_PG_P) == 0 || (Pml4[Pml4Index] & IA32_PG_PMNT) != 0) {\r
+  for (Pml5Index = 0; Pml5Index < Enable5LevelPaging ? (EFI_PAGE_SIZE / sizeof (*Pml4)) : 1; Pml5Index++) {\r
+    if ((Pml5[Pml5Index] & IA32_PG_P) == 0 || (Pml5[Pml5Index] & IA32_PG_PMNT) != 0) {\r
       //\r
-      // If the PML4 entry is not present or is masked, skip it\r
+      // If the PML5 entry is not present or is masked, skip it\r
       //\r
       continue;\r
     }\r
-    Pdpt = (UINT64*)(UINTN)(Pml4[Pml4Index] & ~mAddressEncMask & gPhyMask);\r
-    PML4EIgnore = FALSE;\r
-    for (PdptIndex = 0; PdptIndex < EFI_PAGE_SIZE / sizeof (*Pdpt); PdptIndex++) {\r
-      if ((Pdpt[PdptIndex] & IA32_PG_P) == 0 || (Pdpt[PdptIndex] & IA32_PG_PMNT) != 0) {\r
+    Pml4 = (UINT64*)(UINTN)(Pml5[Pml5Index] & gPhyMask);\r
+    for (Pml4Index = 0; Pml4Index < EFI_PAGE_SIZE / sizeof (*Pml4); Pml4Index++) {\r
+      if ((Pml4[Pml4Index] & IA32_PG_P) == 0 || (Pml4[Pml4Index] & IA32_PG_PMNT) != 0) {\r
         //\r
-        // If the PDPT entry is not present or is masked, skip it\r
+        // If the PML4 entry is not present or is masked, skip it\r
         //\r
-        if ((Pdpt[PdptIndex] & IA32_PG_PMNT) != 0) {\r
-          //\r
-          // If the PDPT entry is masked, we will ignore checking the PML4 entry\r
-          //\r
-          PML4EIgnore = TRUE;\r
-        }\r
         continue;\r
       }\r
-      if ((Pdpt[PdptIndex] & IA32_PG_PS) == 0) {\r
-        //\r
-        // It's not 1-GByte pages entry, it should be a PDPT entry,\r
-        // we will not check PML4 entry more\r
-        //\r
-        PML4EIgnore = TRUE;\r
-        Pdt =  (UINT64*)(UINTN)(Pdpt[PdptIndex] & ~mAddressEncMask & gPhyMask);\r
-        PDPTEIgnore = FALSE;\r
-        for (PdtIndex = 0; PdtIndex < EFI_PAGE_SIZE / sizeof(*Pdt); PdtIndex++) {\r
-          if ((Pdt[PdtIndex] & IA32_PG_P) == 0 || (Pdt[PdtIndex] & IA32_PG_PMNT) != 0) {\r
+      Pdpt = (UINT64*)(UINTN)(Pml4[Pml4Index] & ~mAddressEncMask & gPhyMask);\r
+      PML4EIgnore = FALSE;\r
+      for (PdptIndex = 0; PdptIndex < EFI_PAGE_SIZE / sizeof (*Pdpt); PdptIndex++) {\r
+        if ((Pdpt[PdptIndex] & IA32_PG_P) == 0 || (Pdpt[PdptIndex] & IA32_PG_PMNT) != 0) {\r
+          //\r
+          // If the PDPT entry is not present or is masked, skip it\r
+          //\r
+          if ((Pdpt[PdptIndex] & IA32_PG_PMNT) != 0) {\r
             //\r
-            // If the PD entry is not present or is masked, skip it\r
+            // If the PDPT entry is masked, we will ignore checking the PML4 entry\r
             //\r
-            if ((Pdt[PdtIndex] & IA32_PG_PMNT) != 0) {\r
+            PML4EIgnore = TRUE;\r
+          }\r
+          continue;\r
+        }\r
+        if ((Pdpt[PdptIndex] & IA32_PG_PS) == 0) {\r
+          //\r
+          // It's not 1-GByte pages entry, it should be a PDPT entry,\r
+          // we will not check PML4 entry more\r
+          //\r
+          PML4EIgnore = TRUE;\r
+          Pdt = (UINT64*)(UINTN)(Pdpt[PdptIndex] & ~mAddressEncMask & gPhyMask);\r
+          PDPTEIgnore = FALSE;\r
+          for (PdtIndex = 0; PdtIndex < EFI_PAGE_SIZE / sizeof(*Pdt); PdtIndex++) {\r
+            if ((Pdt[PdtIndex] & IA32_PG_P) == 0 || (Pdt[PdtIndex] & IA32_PG_PMNT) != 0) {\r
+              //\r
+              // If the PD entry is not present or is masked, skip it\r
+              //\r
+              if ((Pdt[PdtIndex] & IA32_PG_PMNT) != 0) {\r
+                //\r
+                // If the PD entry is masked, we will not PDPT entry more\r
+                //\r
+                PDPTEIgnore = TRUE;\r
+              }\r
+              continue;\r
+            }\r
+            if ((Pdt[PdtIndex] & IA32_PG_PS) == 0) {\r
               //\r
-              // If the PD entry is masked, we will not PDPT entry more\r
+              // It's not 2 MByte page table entry, it should be PD entry\r
+              // we will find the entry has the smallest access record value\r
               //\r
               PDPTEIgnore = TRUE;\r
+              Acc = GetAndUpdateAccNum (Pdt + PdtIndex);\r
+              if (Acc < MinAcc) {\r
+                //\r
+                // If the PD entry has the smallest access record value,\r
+                // save the Page address to be released\r
+                //\r
+                MinAcc  = Acc;\r
+                MinPml5 = Pml5Index;\r
+                MinPml4 = Pml4Index;\r
+                MinPdpt = PdptIndex;\r
+                MinPdt  = PdtIndex;\r
+                ReleasePageAddress = Pdt + PdtIndex;\r
+              }\r
             }\r
-            continue;\r
           }\r
-          if ((Pdt[PdtIndex] & IA32_PG_PS) == 0) {\r
+          if (!PDPTEIgnore) {\r
             //\r
-            // It's not 2 MByte page table entry, it should be PD entry\r
-            // we will find the entry has the smallest access record value\r
+            // If this PDPT entry has no PDT entries pointer to 4 KByte pages,\r
+            // it should only has the entries point to 2 MByte Pages\r
             //\r
-            PDPTEIgnore = TRUE;\r
-            Acc = GetAndUpdateAccNum (Pdt + PdtIndex);\r
+            Acc = GetAndUpdateAccNum (Pdpt + PdptIndex);\r
             if (Acc < MinAcc) {\r
               //\r
-              // If the PD entry has the smallest access record value,\r
+              // If the PDPT entry has the smallest access record value,\r
               // save the Page address to be released\r
               //\r
               MinAcc  = Acc;\r
+              MinPml5 = Pml5Index;\r
               MinPml4 = Pml4Index;\r
               MinPdpt = PdptIndex;\r
-              MinPdt  = PdtIndex;\r
-              ReleasePageAddress = Pdt + PdtIndex;\r
+              MinPdt  = (UINTN)-1;\r
+              ReleasePageAddress = Pdpt + PdptIndex;\r
             }\r
           }\r
         }\r
-        if (!PDPTEIgnore) {\r
-          //\r
-          // If this PDPT entry has no PDT entries pointer to 4 KByte pages,\r
-          // it should only has the entries point to 2 MByte Pages\r
-          //\r
-          Acc = GetAndUpdateAccNum (Pdpt + PdptIndex);\r
-          if (Acc < MinAcc) {\r
-            //\r
-            // If the PDPT entry has the smallest access record value,\r
-            // save the Page address to be released\r
-            //\r
-            MinAcc  = Acc;\r
-            MinPml4 = Pml4Index;\r
-            MinPdpt = PdptIndex;\r
-            MinPdt  = (UINTN)-1;\r
-            ReleasePageAddress = Pdpt + PdptIndex;\r
-          }\r
-        }\r
       }\r
-    }\r
-    if (!PML4EIgnore) {\r
-      //\r
-      // If PML4 entry has no the PDPT entry pointer to 2 MByte pages,\r
-      // it should only has the entries point to 1 GByte Pages\r
-      //\r
-      Acc = GetAndUpdateAccNum (Pml4 + Pml4Index);\r
-      if (Acc < MinAcc) {\r
+      if (!PML4EIgnore) {\r
         //\r
-        // If the PML4 entry has the smallest access record value,\r
-        // save the Page address to be released\r
+        // If PML4 entry has no the PDPT entry pointer to 2 MByte pages,\r
+        // it should only has the entries point to 1 GByte Pages\r
         //\r
-        MinAcc  = Acc;\r
-        MinPml4 = Pml4Index;\r
-        MinPdpt = (UINTN)-1;\r
-        MinPdt  = (UINTN)-1;\r
-        ReleasePageAddress = Pml4 + Pml4Index;\r
+        Acc = GetAndUpdateAccNum (Pml4 + Pml4Index);\r
+        if (Acc < MinAcc) {\r
+          //\r
+          // If the PML4 entry has the smallest access record value,\r
+          // save the Page address to be released\r
+          //\r
+          MinAcc  = Acc;\r
+          MinPml5 = Pml5Index;\r
+          MinPml4 = Pml4Index;\r
+          MinPdpt = (UINTN)-1;\r
+          MinPdt  = (UINTN)-1;\r
+          ReleasePageAddress = Pml4 + Pml4Index;\r
+        }\r
       }\r
     }\r
   }\r
@@ -588,6 +705,7 @@ ReclaimPages (
       //\r
       // If 4 KByte Page Table is released, check the PDPT entry\r
       //\r
+      Pml4 = (UINT64 *) (UINTN) (Pml5[MinPml5] & gPhyMask);\r
       Pdpt = (UINT64*)(UINTN)(Pml4[MinPml4] & ~mAddressEncMask & gPhyMask);\r
       SubEntriesNum = GetSubEntriesNum(Pdpt + MinPdpt);\r
       if (SubEntriesNum == 0) {\r
@@ -679,7 +797,7 @@ SmiDefaultPFHandler (
   )\r
 {\r
   UINT64                            *PageTable;\r
-  UINT64                            *Pml4;\r
+  UINT64                            *PageTableTop;\r
   UINT64                            PFAddress;\r
   UINTN                             StartBit;\r
   UINTN                             EndBit;\r
@@ -690,6 +808,8 @@ SmiDefaultPFHandler (
   UINTN                             PageAttribute;\r
   EFI_STATUS                        Status;\r
   UINT64                            *UpperEntry;\r
+  BOOLEAN                           Enable5LevelPaging;\r
+  IA32_CR4                          Cr4;\r
 \r
   //\r
   // Set default SMM page attribute\r
@@ -699,9 +819,12 @@ SmiDefaultPFHandler (
   PageAttribute = 0;\r
 \r
   EndBit = 0;\r
-  Pml4 = (UINT64*)(AsmReadCr3 () & gPhyMask);\r
+  PageTableTop = (UINT64*)(AsmReadCr3 () & gPhyMask);\r
   PFAddress = AsmReadCr2 ();\r
 \r
+  Cr4.UintN = AsmReadCr4 ();\r
+  Enable5LevelPaging = (BOOLEAN) (Cr4.Bits.LA57 != 0);\r
+\r
   Status = GetPlatformPageTableAttribute (PFAddress, &PageSize, &NumOfPages, &PageAttribute);\r
   //\r
   // If platform not support page table attribute, set default SMM page attribute\r
@@ -755,9 +878,9 @@ SmiDefaultPFHandler (
   }\r
 \r
   for (Index = 0; Index < NumOfPages; Index++) {\r
-    PageTable  = Pml4;\r
+    PageTable  = PageTableTop;\r
     UpperEntry = NULL;\r
-    for (StartBit = 39; StartBit > EndBit; StartBit -= 9) {\r
+    for (StartBit = Enable5LevelPaging ? 48 : 39; StartBit > EndBit; StartBit -= 9) {\r
       PTIndex = BitFieldRead64 (PFAddress, StartBit, StartBit + 8);\r
       if ((PageTable[PTIndex] & IA32_PG_P) == 0) {\r
         //\r
@@ -941,13 +1064,20 @@ SetPageTableAttributes (
   UINTN                 Index2;\r
   UINTN                 Index3;\r
   UINTN                 Index4;\r
+  UINTN                 Index5;\r
   UINT64                *L1PageTable;\r
   UINT64                *L2PageTable;\r
   UINT64                *L3PageTable;\r
   UINT64                *L4PageTable;\r
+  UINT64                *L5PageTable;\r
   BOOLEAN               IsSplitted;\r
   BOOLEAN               PageTableSplitted;\r
   BOOLEAN               CetEnabled;\r
+  IA32_CR4              Cr4;\r
+  BOOLEAN               Enable5LevelPaging;\r
+\r
+  Cr4.UintN = AsmReadCr4 ();\r
+  Enable5LevelPaging = (BOOLEAN) (Cr4.Bits.LA57 == 1);\r
 \r
   //\r
   // Don't do this if\r
@@ -991,44 +1121,59 @@ SetPageTableAttributes (
   do {\r
     DEBUG ((DEBUG_INFO, "Start...\n"));\r
     PageTableSplitted = FALSE;\r
-\r
-    L4PageTable = (UINT64 *)GetPageTableBase ();\r
-    SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS)(UINTN)L4PageTable, SIZE_4KB, EFI_MEMORY_RO, &IsSplitted);\r
-    PageTableSplitted = (PageTableSplitted || IsSplitted);\r
-\r
-    for (Index4 = 0; Index4 < SIZE_4KB/sizeof(UINT64); Index4++) {\r
-      L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & ~mAddressEncMask & PAGING_4K_ADDRESS_MASK_64);\r
-      if (L3PageTable == NULL) {\r
-        continue;\r
-      }\r
-\r
-      SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS)(UINTN)L3PageTable, SIZE_4KB, EFI_MEMORY_RO, &IsSplitted);\r
+    L5PageTable = NULL;\r
+    if (Enable5LevelPaging) {\r
+      L5PageTable = (UINT64 *)GetPageTableBase ();\r
+      SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS)(UINTN)L5PageTable, SIZE_4KB, EFI_MEMORY_RO, &IsSplitted);\r
       PageTableSplitted = (PageTableSplitted || IsSplitted);\r
+    }\r
 \r
-      for (Index3 = 0; Index3 < SIZE_4KB/sizeof(UINT64); Index3++) {\r
-        if ((L3PageTable[Index3] & IA32_PG_PS) != 0) {\r
-          // 1G\r
+    for (Index5 = 0; Index5 < (Enable5LevelPaging ? SIZE_4KB/sizeof(UINT64) : 1); Index5++) {\r
+      if (Enable5LevelPaging) {\r
+        L4PageTable = (UINT64 *)(UINTN)(L5PageTable[Index5] & ~mAddressEncMask & PAGING_4K_ADDRESS_MASK_64);\r
+        if (L4PageTable == NULL) {\r
           continue;\r
         }\r
-        L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & ~mAddressEncMask & PAGING_4K_ADDRESS_MASK_64);\r
-        if (L2PageTable == NULL) {\r
+      } else {\r
+        L4PageTable = (UINT64 *)GetPageTableBase ();\r
+      }\r
+      SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS)(UINTN)L4PageTable, SIZE_4KB, EFI_MEMORY_RO, &IsSplitted);\r
+      PageTableSplitted = (PageTableSplitted || IsSplitted);\r
+\r
+      for (Index4 = 0; Index4 < SIZE_4KB/sizeof(UINT64); Index4++) {\r
+        L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & ~mAddressEncMask & PAGING_4K_ADDRESS_MASK_64);\r
+        if (L3PageTable == NULL) {\r
           continue;\r
         }\r
 \r
-        SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS)(UINTN)L2PageTable, SIZE_4KB, EFI_MEMORY_RO, &IsSplitted);\r
+        SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS)(UINTN)L3PageTable, SIZE_4KB, EFI_MEMORY_RO, &IsSplitted);\r
         PageTableSplitted = (PageTableSplitted || IsSplitted);\r
 \r
-        for (Index2 = 0; Index2 < SIZE_4KB/sizeof(UINT64); Index2++) {\r
-          if ((L2PageTable[Index2] & IA32_PG_PS) != 0) {\r
-            // 2M\r
+        for (Index3 = 0; Index3 < SIZE_4KB/sizeof(UINT64); Index3++) {\r
+          if ((L3PageTable[Index3] & IA32_PG_PS) != 0) {\r
+            // 1G\r
             continue;\r
           }\r
-          L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & ~mAddressEncMask & PAGING_4K_ADDRESS_MASK_64);\r
-          if (L1PageTable == NULL) {\r
+          L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & ~mAddressEncMask & PAGING_4K_ADDRESS_MASK_64);\r
+          if (L2PageTable == NULL) {\r
             continue;\r
           }\r
-          SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS)(UINTN)L1PageTable, SIZE_4KB, EFI_MEMORY_RO, &IsSplitted);\r
+\r
+          SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS)(UINTN)L2PageTable, SIZE_4KB, EFI_MEMORY_RO, &IsSplitted);\r
           PageTableSplitted = (PageTableSplitted || IsSplitted);\r
+\r
+          for (Index2 = 0; Index2 < SIZE_4KB/sizeof(UINT64); Index2++) {\r
+            if ((L2PageTable[Index2] & IA32_PG_PS) != 0) {\r
+              // 2M\r
+              continue;\r
+            }\r
+            L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & ~mAddressEncMask & PAGING_4K_ADDRESS_MASK_64);\r
+            if (L1PageTable == NULL) {\r
+              continue;\r
+            }\r
+            SmmSetMemoryAttributesEx ((EFI_PHYSICAL_ADDRESS)(UINTN)L1PageTable, SIZE_4KB, EFI_MEMORY_RO, &IsSplitted);\r
+            PageTableSplitted = (PageTableSplitted || IsSplitted);\r
+          }\r
         }\r
       }\r
     }\r

diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/X64/SmiEntry.nasm b/UefiCpuPkg/PiSmmCpuDxeSmm/X64/SmiEntry.nasm
index 741e4b7da2bd1061cd9b3bbae350aeeb22f24f48..271492a9d7220b13c8de2b329e2788a60da26ac3 100644 (file)
--- a/UefiCpuPkg/PiSmmCpuDxeSmm/X64/SmiEntry.nasm
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/X64/SmiEntry.nasm
@@ -69,6 +69,7 @@ extern ASM_PFX(mXdSupported)
 global ASM_PFX(gPatchXdSupported)\r
 global ASM_PFX(gPatchSmiStack)\r
 global ASM_PFX(gPatchSmiCr3)\r
+global ASM_PFX(gPatch5LevelPagingSupport)\r
 global ASM_PFX(gcSmiHandlerTemplate)\r
 global ASM_PFX(gcSmiHandlerSize)\r
 \r
@@ -124,6 +125,17 @@ ProtFlatMode:
 ASM_PFX(gPatchSmiCr3):\r
     mov     cr3, rax\r
     mov     eax, 0x668                   ; as cr4.PGE is not set here, refresh cr3\r
+\r
+    mov     cl, strict byte 0            ; source operand will be patched\r
+ASM_PFX(gPatch5LevelPagingSupport):\r
+    cmp     cl, 0\r
+    je      SkipEnable5LevelPaging\r
+    ;\r
+    ; Enable 5-Level Paging bit\r
+    ;\r
+    bts     eax, 12                     ; Set LA57 bit (bit #12)\r
+SkipEnable5LevelPaging:\r
+\r
     mov     cr4, rax                    ; in PreModifyMtrrs() to flush TLB.\r
 ; Load TSS\r
     sub     esp, 8                      ; reserve room in stack\r

diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/X64/SmmProfileArch.c b/UefiCpuPkg/PiSmmCpuDxeSmm/X64/SmmProfileArch.c
index e7c78d36fc5fa8f0d345a9eab7695a8eea620baa..63bae5a9137cbbff9cf2617b1f5825429a6eb141 100644 (file)
--- a/UefiCpuPkg/PiSmmCpuDxeSmm/X64/SmmProfileArch.c
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/X64/SmmProfileArch.c
@@ -1,7 +1,7 @@
 /** @file\r
 X64 processor specific functions to enable SMM profile.\r
 \r
-Copyright (c) 2012 - 2016, Intel Corporation. All rights reserved.<BR>\r
+Copyright (c) 2012 - 2019, Intel Corporation. All rights reserved.<BR>\r
 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>\r
 \r
 SPDX-License-Identifier: BSD-2-Clause-Patent\r
@@ -147,9 +147,14 @@ RestorePageTableAbove4G (
   BOOLEAN       Existed;\r
   UINTN         Index;\r
   UINTN         PFIndex;\r
+  IA32_CR4      Cr4;\r
+  BOOLEAN       Enable5LevelPaging;\r
 \r
   ASSERT ((PageTable != NULL) && (IsValidPFAddress != NULL));\r
 \r
+  Cr4.UintN = AsmReadCr4 ();\r
+  Enable5LevelPaging = (BOOLEAN) (Cr4.Bits.LA57 == 1);\r
+\r
   //\r
   // If page fault address is 4GB above.\r
   //\r
@@ -161,38 +166,48 @@ RestorePageTableAbove4G (
   //\r
   Existed = FALSE;\r
   PageTable = (UINT64*)(AsmReadCr3 () & PHYSICAL_ADDRESS_MASK);\r
-  PTIndex = BitFieldRead64 (PFAddress, 39, 47);\r
-  if ((PageTable[PTIndex] & IA32_PG_P) != 0) {\r
-    // PML4E\r
-    PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
-    PTIndex = BitFieldRead64 (PFAddress, 30, 38);\r
+  PTIndex = 0;\r
+  if (Enable5LevelPaging) {\r
+    PTIndex = BitFieldRead64 (PFAddress, 48, 56);\r
+  }\r
+  if ((!Enable5LevelPaging) || ((PageTable[PTIndex] & IA32_PG_P) != 0)) {\r
+    // PML5E\r
+    if (Enable5LevelPaging) {\r
+      PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
+    }\r
+    PTIndex = BitFieldRead64 (PFAddress, 39, 47);\r
     if ((PageTable[PTIndex] & IA32_PG_P) != 0) {\r
-      // PDPTE\r
+      // PML4E\r
       PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
-      PTIndex = BitFieldRead64 (PFAddress, 21, 29);\r
-      // PD\r
-      if ((PageTable[PTIndex] & IA32_PG_PS) != 0) {\r
-        //\r
-        // 2MB page\r
-        //\r
-        Address = (UINT64)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
-        if ((Address & ~((1ull << 21) - 1)) == ((PFAddress & PHYSICAL_ADDRESS_MASK & ~((1ull << 21) - 1)))) {\r
-          Existed = TRUE;\r
-        }\r
-      } else {\r
-        //\r
-        // 4KB page\r
-        //\r
-        PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask& PHYSICAL_ADDRESS_MASK);\r
-        if (PageTable != 0) {\r
+      PTIndex = BitFieldRead64 (PFAddress, 30, 38);\r
+      if ((PageTable[PTIndex] & IA32_PG_P) != 0) {\r
+        // PDPTE\r
+        PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
+        PTIndex = BitFieldRead64 (PFAddress, 21, 29);\r
+        // PD\r
+        if ((PageTable[PTIndex] & IA32_PG_PS) != 0) {\r
           //\r
-          // When there is a valid entry to map to 4KB page, need not create a new entry to map 2MB.\r
+          // 2MB page\r
           //\r
-          PTIndex = BitFieldRead64 (PFAddress, 12, 20);\r
           Address = (UINT64)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
-          if ((Address & ~((1ull << 12) - 1)) == (PFAddress & PHYSICAL_ADDRESS_MASK & ~((1ull << 12) - 1))) {\r
+          if ((Address & ~((1ull << 21) - 1)) == ((PFAddress & PHYSICAL_ADDRESS_MASK & ~((1ull << 21) - 1)))) {\r
             Existed = TRUE;\r
           }\r
+        } else {\r
+          //\r
+          // 4KB page\r
+          //\r
+          PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask& PHYSICAL_ADDRESS_MASK);\r
+          if (PageTable != 0) {\r
+            //\r
+            // When there is a valid entry to map to 4KB page, need not create a new entry to map 2MB.\r
+            //\r
+            PTIndex = BitFieldRead64 (PFAddress, 12, 20);\r
+            Address = (UINT64)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
+            if ((Address & ~((1ull << 12) - 1)) == (PFAddress & PHYSICAL_ADDRESS_MASK & ~((1ull << 12) - 1))) {\r
+              Existed = TRUE;\r
+            }\r
+          }\r
         }\r
       }\r
     }\r
@@ -221,6 +236,11 @@ RestorePageTableAbove4G (
     //\r
     PageTable = (UINT64*)(AsmReadCr3 () & PHYSICAL_ADDRESS_MASK);\r
     PFAddress = AsmReadCr2 ();\r
+    // PML5E\r
+    if (Enable5LevelPaging) {\r
+      PTIndex = BitFieldRead64 (PFAddress, 48, 56);\r
+      PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r
+    }\r
     // PML4E\r
     PTIndex = BitFieldRead64 (PFAddress, 39, 47);\r
     PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & ~mAddressEncMask & PHYSICAL_ADDRESS_MASK);\r