[mirror_edk2.git] / UefiCpuPkg / CpuDxe / CpuPageTable.c

/** @file\r
  Page table management support.\r
\r
  Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>\r
  Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>\r
\r
  This program and the accompanying materials\r
  are licensed and made available under the terms and conditions of the BSD License\r
  which accompanies this distribution.  The full text of the license may be found at\r
  http://opensource.org/licenses/bsd-license.php\r
\r
  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
\r
**/\r
\r
#include <Base.h>\r
#include <Uefi.h>\r
#include <Library/BaseLib.h>\r
#include <Library/CpuLib.h>\r
#include <Library/BaseMemoryLib.h>\r
#include <Library/MemoryAllocationLib.h>\r
#include <Library/DebugLib.h>\r
#include <Library/UefiBootServicesTableLib.h>\r
#include <Protocol/MpService.h>\r
\r
#include "CpuDxe.h"\r
#include "CpuPageTable.h"\r
\r
///\r
/// Page Table Entry\r
///\r
#define IA32_PG_P                   BIT0\r
#define IA32_PG_RW                  BIT1\r
#define IA32_PG_U                   BIT2\r
#define IA32_PG_WT                  BIT3\r
#define IA32_PG_CD                  BIT4\r
#define IA32_PG_A                   BIT5\r
#define IA32_PG_D                   BIT6\r
#define IA32_PG_PS                  BIT7\r
#define IA32_PG_PAT_2M              BIT12\r
#define IA32_PG_PAT_4K              IA32_PG_PS\r
#define IA32_PG_PMNT                BIT62\r
#define IA32_PG_NX                  BIT63\r
\r
#define PAGE_ATTRIBUTE_BITS         (IA32_PG_D | IA32_PG_A | IA32_PG_U | IA32_PG_RW | IA32_PG_P)\r
//\r
// Bits 1, 2, 5, 6 are reserved in the IA32 PAE PDPTE\r
// X64 PAE PDPTE does not have such restriction\r
//\r
#define IA32_PAE_PDPTE_ATTRIBUTE_BITS    (IA32_PG_P)\r
\r
#define PAGE_PROGATE_BITS           (IA32_PG_NX | PAGE_ATTRIBUTE_BITS)\r
\r
#define PAGING_4K_MASK  0xFFF\r
#define PAGING_2M_MASK  0x1FFFFF\r
#define PAGING_1G_MASK  0x3FFFFFFF\r
\r
#define PAGING_PAE_INDEX_MASK  0x1FF\r
\r
#define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull\r
#define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull\r
#define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull\r
\r
typedef enum {\r
  PageNone,\r
  Page4K,\r
  Page2M,\r
  Page1G,\r
} PAGE_ATTRIBUTE;\r
\r
typedef struct {\r
  PAGE_ATTRIBUTE   Attribute;\r
  UINT64           Length;\r
  UINT64           AddressMask;\r
} PAGE_ATTRIBUTE_TABLE;\r
\r
typedef enum {\r
  PageActionAssign,\r
  PageActionSet,\r
  PageActionClear,\r
} PAGE_ACTION;\r
\r
PAGE_ATTRIBUTE_TABLE mPageAttributeTable[] = {\r
  {Page4K,  SIZE_4KB, PAGING_4K_ADDRESS_MASK_64},\r
  {Page2M,  SIZE_2MB, PAGING_2M_ADDRESS_MASK_64},\r
  {Page1G,  SIZE_1GB, PAGING_1G_ADDRESS_MASK_64},\r
};\r
\r
PAGE_TABLE_POOL   *mPageTablePool = NULL;\r
\r
/**\r
  Enable write protection function for AP.\r
\r
  @param[in,out] Buffer  The pointer to private data buffer.\r
**/\r
VOID\r
EFIAPI\r
SyncCpuEnableWriteProtection (\r
  IN OUT VOID *Buffer\r
  )\r
{\r
  AsmWriteCr0 (AsmReadCr0 () | BIT16);\r
}\r
\r
/**\r
  CpuFlushTlb function for AP.\r
\r
  @param[in,out] Buffer  The pointer to private data buffer.\r
**/\r
VOID\r
EFIAPI\r
SyncCpuFlushTlb (\r
  IN OUT VOID *Buffer\r
  )\r
{\r
  CpuFlushTlb();\r
}\r
\r
/**\r
  Sync memory page attributes for AP.\r
\r
  @param[in] Procedure            A pointer to the function to be run on enabled APs of\r
                                  the system.\r
**/\r
VOID\r
SyncMemoryPageAttributesAp (\r
  IN EFI_AP_PROCEDURE            Procedure\r
  )\r
{\r
  EFI_STATUS                Status;\r
  EFI_MP_SERVICES_PROTOCOL  *MpService;\r
\r
  Status = gBS->LocateProtocol (\r
                  &gEfiMpServiceProtocolGuid,\r
                  NULL,\r
                  (VOID **)&MpService\r
                  );\r
  //\r
  // Synchronize the update with all APs\r
  //\r
  if (!EFI_ERROR (Status)) {\r
    Status = MpService->StartupAllAPs (\r
                          MpService,          // This\r
                          Procedure,          // Procedure\r
                          FALSE,              // SingleThread\r
                          NULL,               // WaitEvent\r
                          0,                  // TimeoutInMicrosecsond\r
                          NULL,               // ProcedureArgument\r
                          NULL                // FailedCpuList\r
                          );\r
    ASSERT (Status == EFI_SUCCESS || Status == EFI_NOT_STARTED || Status == EFI_NOT_READY);\r
  }\r
}\r
\r
/**\r
  Return current paging context.\r
\r
  @param[in,out]  PagingContext     The paging context.\r
**/\r
VOID\r
GetCurrentPagingContext (\r
  IN OUT PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext\r
  )\r
{\r
  UINT32                         RegEax;\r
  UINT32                         RegEdx;\r
\r
  ZeroMem(PagingContext, sizeof(*PagingContext));\r
  if (sizeof(UINTN) == sizeof(UINT64)) {\r
    PagingContext->MachineType = IMAGE_FILE_MACHINE_X64;\r
  } else {\r
    PagingContext->MachineType = IMAGE_FILE_MACHINE_I386;\r
  }\r
  if ((AsmReadCr0 () & BIT31) != 0) {\r
    PagingContext->ContextData.X64.PageTableBase = (AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64);\r
  } else {\r
    PagingContext->ContextData.X64.PageTableBase = 0;\r
  }\r
\r
  if ((AsmReadCr4 () & BIT4) != 0) {\r
    PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PSE;\r
  }\r
  if ((AsmReadCr4 () & BIT5) != 0) {\r
    PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE;\r
  }\r
  if ((AsmReadCr0 () & BIT16) != 0) {\r
    PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_WP_ENABLE;\r
  }\r
\r
  AsmCpuid (0x80000000, &RegEax, NULL, NULL, NULL);\r
  if (RegEax > 0x80000000) {\r
    AsmCpuid (0x80000001, NULL, NULL, NULL, &RegEdx);\r
    if ((RegEdx & BIT20) != 0) {\r
      // XD supported\r
      if ((AsmReadMsr64 (0xC0000080) & BIT11) != 0) {\r
        // XD activated\r
        PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED;\r
      }\r
    }\r
    if ((RegEdx & BIT26) != 0) {\r
      PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAGE_1G_SUPPORT;\r
    }\r
  }\r
}\r
\r
/**\r
  Return length according to page attributes.\r
\r
  @param[in]  PageAttributes   The page attribute of the page entry.\r
\r
  @return The length of page entry.\r
**/\r
UINTN\r
PageAttributeToLength (\r
  IN PAGE_ATTRIBUTE  PageAttribute\r
  )\r
{\r
  UINTN  Index;\r
  for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {\r
    if (PageAttribute == mPageAttributeTable[Index].Attribute) {\r
      return (UINTN)mPageAttributeTable[Index].Length;\r
    }\r
  }\r
  return 0;\r
}\r
\r
/**\r
  Return address mask according to page attributes.\r
\r
  @param[in]  PageAttributes   The page attribute of the page entry.\r
\r
  @return The address mask of page entry.\r
**/\r
UINTN\r
PageAttributeToMask (\r
  IN PAGE_ATTRIBUTE  PageAttribute\r
  )\r
{\r
  UINTN  Index;\r
  for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {\r
    if (PageAttribute == mPageAttributeTable[Index].Attribute) {\r
      return (UINTN)mPageAttributeTable[Index].AddressMask;\r
    }\r
  }\r
  return 0;\r
}\r
\r
/**\r
  Return page table entry to match the address.\r
\r
  @param[in]  PagingContext     The paging context.\r
  @param[in]  Address           The address to be checked.\r
  @param[out] PageAttributes    The page attribute of the page entry.\r
\r
  @return The page entry.\r
**/\r
VOID *\r
GetPageTableEntry (\r
  IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext,\r
  IN  PHYSICAL_ADDRESS                  Address,\r
  OUT PAGE_ATTRIBUTE                    *PageAttribute\r
  )\r
{\r
  UINTN                 Index1;\r
  UINTN                 Index2;\r
  UINTN                 Index3;\r
  UINTN                 Index4;\r
  UINT64                *L1PageTable;\r
  UINT64                *L2PageTable;\r
  UINT64                *L3PageTable;\r
  UINT64                *L4PageTable;\r
  UINT64                AddressEncMask;\r
\r
  ASSERT (PagingContext != NULL);\r
\r
  Index4 = ((UINTN)RShiftU64 (Address, 39)) & PAGING_PAE_INDEX_MASK;\r
  Index3 = ((UINTN)Address >> 30) & PAGING_PAE_INDEX_MASK;\r
  Index2 = ((UINTN)Address >> 21) & PAGING_PAE_INDEX_MASK;\r
  Index1 = ((UINTN)Address >> 12) & PAGING_PAE_INDEX_MASK;\r
\r
  // Make sure AddressEncMask is contained to smallest supported address field.\r
  //\r
  AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;\r
\r
  if (PagingContext->MachineType == IMAGE_FILE_MACHINE_X64) {\r
    L4PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.X64.PageTableBase;\r
    if (L4PageTable[Index4] == 0) {\r
      *PageAttribute = PageNone;\r
      return NULL;\r
    }\r
\r
    L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);\r
  } else {\r
    ASSERT((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0);\r
    L3PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.Ia32.PageTableBase;\r
  }\r
  if (L3PageTable[Index3] == 0) {\r
    *PageAttribute = PageNone;\r
    return NULL;\r
  }\r
  if ((L3PageTable[Index3] & IA32_PG_PS) != 0) {\r
    // 1G\r
    *PageAttribute = Page1G;\r
    return &L3PageTable[Index3];\r
  }\r
\r
  L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);\r
  if (L2PageTable[Index2] == 0) {\r
    *PageAttribute = PageNone;\r
    return NULL;\r
  }\r
  if ((L2PageTable[Index2] & IA32_PG_PS) != 0) {\r
    // 2M\r
    *PageAttribute = Page2M;\r
    return &L2PageTable[Index2];\r
  }\r
\r
  // 4k\r
  L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);\r
  if ((L1PageTable[Index1] == 0) && (Address != 0)) {\r
    *PageAttribute = PageNone;\r
    return NULL;\r
  }\r
  *PageAttribute = Page4K;\r
  return &L1PageTable[Index1];\r
}\r
\r
/**\r
  Return memory attributes of page entry.\r
\r
  @param[in]  PageEntry        The page entry.\r
\r
  @return Memory attributes of page entry.\r
**/\r
UINT64\r
GetAttributesFromPageEntry (\r
  IN  UINT64                            *PageEntry\r
  )\r
{\r
  UINT64  Attributes;\r
  Attributes = 0;\r
  if ((*PageEntry & IA32_PG_P) == 0) {\r
    Attributes |= EFI_MEMORY_RP;\r
  }\r
  if ((*PageEntry & IA32_PG_RW) == 0) {\r
    Attributes |= EFI_MEMORY_RO;\r
  }\r
  if ((*PageEntry & IA32_PG_NX) != 0) {\r
    Attributes |= EFI_MEMORY_XP;\r
  }\r
  return Attributes;\r
}\r
\r
/**\r
  Modify memory attributes of page entry.\r
\r
  @param[in]  PagingContext    The paging context.\r
  @param[in]  PageEntry        The page entry.\r
  @param[in]  Attributes       The bit mask of attributes to modify for the memory region.\r
  @param[in]  PageAction       The page action.\r
  @param[out] IsModified       TRUE means page table modified. FALSE means page table not modified.\r
**/\r
VOID\r
ConvertPageEntryAttribute (\r
  IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext,\r
  IN  UINT64                            *PageEntry,\r
  IN  UINT64                            Attributes,\r
  IN  PAGE_ACTION                       PageAction,\r
  OUT BOOLEAN                           *IsModified\r
  )\r
{\r
  UINT64  CurrentPageEntry;\r
  UINT64  NewPageEntry;\r
\r
  CurrentPageEntry = *PageEntry;\r
  NewPageEntry = CurrentPageEntry;\r
  if ((Attributes & EFI_MEMORY_RP) != 0) {\r
    switch (PageAction) {\r
    case PageActionAssign:\r
    case PageActionSet:\r
      NewPageEntry &= ~(UINT64)IA32_PG_P;\r
      break;\r
    case PageActionClear:\r
      NewPageEntry |= IA32_PG_P;\r
      break;\r
    }\r
  } else {\r
    switch (PageAction) {\r
    case PageActionAssign:\r
      NewPageEntry |= IA32_PG_P;\r
      break;\r
    case PageActionSet:\r
    case PageActionClear:\r
      break;\r
    }\r
  }\r
  if ((Attributes & EFI_MEMORY_RO) != 0) {\r
    switch (PageAction) {\r
    case PageActionAssign:\r
    case PageActionSet:\r
      NewPageEntry &= ~(UINT64)IA32_PG_RW;\r
      break;\r
    case PageActionClear:\r
      NewPageEntry |= IA32_PG_RW;\r
      break;\r
    }\r
  } else {\r
    switch (PageAction) {\r
    case PageActionAssign:\r
      NewPageEntry |= IA32_PG_RW;\r
      break;\r
    case PageActionSet:\r
    case PageActionClear:\r
      break;\r
    }\r
  }\r
  if ((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED) != 0) {\r
    if ((Attributes & EFI_MEMORY_XP) != 0) {\r
      switch (PageAction) {\r
      case PageActionAssign:\r
      case PageActionSet:\r
        NewPageEntry |= IA32_PG_NX;\r
        break;\r
      case PageActionClear:\r
        NewPageEntry &= ~IA32_PG_NX;\r
        break;\r
      }\r
    } else {\r
      switch (PageAction) {\r
      case PageActionAssign:\r
        NewPageEntry &= ~IA32_PG_NX;\r
        break;\r
      case PageActionSet:\r
      case PageActionClear:\r
        break;\r
      }\r
    }\r
  }\r
  *PageEntry = NewPageEntry;\r
  if (CurrentPageEntry != NewPageEntry) {\r
    *IsModified = TRUE;\r
    DEBUG ((DEBUG_VERBOSE, "ConvertPageEntryAttribute 0x%lx", CurrentPageEntry));\r
    DEBUG ((DEBUG_VERBOSE, "->0x%lx\n", NewPageEntry));\r
  } else {\r
    *IsModified = FALSE;\r
  }\r
}\r
\r
/**\r
  This function returns if there is need to split page entry.\r
\r
  @param[in]  BaseAddress      The base address to be checked.\r
  @param[in]  Length           The length to be checked.\r
  @param[in]  PageEntry        The page entry to be checked.\r
  @param[in]  PageAttribute    The page attribute of the page entry.\r
\r
  @retval SplitAttributes on if there is need to split page entry.\r
**/\r
PAGE_ATTRIBUTE\r
NeedSplitPage (\r
  IN  PHYSICAL_ADDRESS                  BaseAddress,\r
  IN  UINT64                            Length,\r
  IN  UINT64                            *PageEntry,\r
  IN  PAGE_ATTRIBUTE                    PageAttribute\r
  )\r
{\r
  UINT64                PageEntryLength;\r
\r
  PageEntryLength = PageAttributeToLength (PageAttribute);\r
\r
  if (((BaseAddress & (PageEntryLength - 1)) == 0) && (Length >= PageEntryLength)) {\r
    return PageNone;\r
  }\r
\r
  if (((BaseAddress & PAGING_2M_MASK) != 0) || (Length < SIZE_2MB)) {\r
    return Page4K;\r
  }\r
\r
  return Page2M;\r
}\r
\r
/**\r
  This function splits one page entry to small page entries.\r
\r
  @param[in]  PageEntry         The page entry to be splitted.\r
  @param[in]  PageAttribute     The page attribute of the page entry.\r
  @param[in]  SplitAttribute    How to split the page entry.\r
  @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.\r
\r
  @retval RETURN_SUCCESS            The page entry is splitted.\r
  @retval RETURN_UNSUPPORTED        The page entry does not support to be splitted.\r
  @retval RETURN_OUT_OF_RESOURCES   No resource to split page entry.\r
**/\r
RETURN_STATUS\r
SplitPage (\r
  IN  UINT64                            *PageEntry,\r
  IN  PAGE_ATTRIBUTE                    PageAttribute,\r
  IN  PAGE_ATTRIBUTE                    SplitAttribute,\r
  IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc\r
  )\r
{\r
  UINT64   BaseAddress;\r
  UINT64   *NewPageEntry;\r
  UINTN    Index;\r
  UINT64   AddressEncMask;\r
\r
  ASSERT (PageAttribute == Page2M || PageAttribute == Page1G);\r
\r
  ASSERT (AllocatePagesFunc != NULL);\r
\r
  // Make sure AddressEncMask is contained to smallest supported address field.\r
  //\r
  AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;\r
\r
  if (PageAttribute == Page2M) {\r
    //\r
    // Split 2M to 4K\r
    //\r
    ASSERT (SplitAttribute == Page4K);\r
    if (SplitAttribute == Page4K) {\r
      NewPageEntry = AllocatePagesFunc (1);\r
      DEBUG ((DEBUG_INFO, "Split - 0x%x\n", NewPageEntry));\r
      if (NewPageEntry == NULL) {\r
        return RETURN_OUT_OF_RESOURCES;\r
      }\r
      BaseAddress = *PageEntry & ~AddressEncMask & PAGING_2M_ADDRESS_MASK_64;\r
      for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {\r
        NewPageEntry[Index] = (BaseAddress + SIZE_4KB * Index) | AddressEncMask | ((*PageEntry) & PAGE_PROGATE_BITS);\r
      }\r
      (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | ((*PageEntry) & PAGE_ATTRIBUTE_BITS);\r
      return RETURN_SUCCESS;\r
    } else {\r
      return RETURN_UNSUPPORTED;\r
    }\r
  } else if (PageAttribute == Page1G) {\r
    //\r
    // Split 1G to 2M\r
    // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.\r
    //\r
    ASSERT (SplitAttribute == Page2M || SplitAttribute == Page4K);\r
    if ((SplitAttribute == Page2M || SplitAttribute == Page4K)) {\r
      NewPageEntry = AllocatePagesFunc (1);\r
      DEBUG ((DEBUG_INFO, "Split - 0x%x\n", NewPageEntry));\r
      if (NewPageEntry == NULL) {\r
        return RETURN_OUT_OF_RESOURCES;\r
      }\r
      BaseAddress = *PageEntry & ~AddressEncMask  & PAGING_1G_ADDRESS_MASK_64;\r
      for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {\r
        NewPageEntry[Index] = (BaseAddress + SIZE_2MB * Index) | AddressEncMask | IA32_PG_PS | ((*PageEntry) & PAGE_PROGATE_BITS);\r
      }\r
      (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | ((*PageEntry) & PAGE_ATTRIBUTE_BITS);\r
      return RETURN_SUCCESS;\r
    } else {\r
      return RETURN_UNSUPPORTED;\r
    }\r
  } else {\r
    return RETURN_UNSUPPORTED;\r
  }\r
}\r
\r
/**\r
 Check the WP status in CR0 register. This bit is used to lock or unlock write\r
 access to pages marked as read-only.\r
\r
  @retval TRUE    Write protection is enabled.\r
  @retval FALSE   Write protection is disabled.\r
**/\r
BOOLEAN\r
IsReadOnlyPageWriteProtected (\r
  VOID\r
  )\r
{\r
  return ((AsmReadCr0 () & BIT16) != 0);\r
}\r
\r
/**\r
  Disable write protection function for AP.\r
\r
  @param[in,out] Buffer  The pointer to private data buffer.\r
**/\r
VOID\r
EFIAPI\r
SyncCpuDisableWriteProtection (\r
  IN OUT VOID *Buffer\r
  )\r
{\r
  AsmWriteCr0 (AsmReadCr0() & ~BIT16);\r
}\r
\r
/**\r
 Disable Write Protect on pages marked as read-only.\r
**/\r
VOID\r
DisableReadOnlyPageWriteProtect (\r
  VOID\r
  )\r
{\r
  AsmWriteCr0 (AsmReadCr0() & ~BIT16);\r
  SyncMemoryPageAttributesAp (SyncCpuDisableWriteProtection);\r
}\r
\r
/**\r
 Enable Write Protect on pages marked as read-only.\r
**/\r
VOID\r
EnableReadOnlyPageWriteProtect (\r
  VOID\r
  )\r
{\r
  AsmWriteCr0 (AsmReadCr0() | BIT16);\r
  SyncMemoryPageAttributesAp (SyncCpuEnableWriteProtection);\r
}\r
\r
/**\r
  This function modifies the page attributes for the memory region specified by BaseAddress and\r
  Length from their current attributes to the attributes specified by Attributes.\r
\r
  Caller should make sure BaseAddress and Length is at page boundary.\r
\r
  @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.\r
  @param[in]  BaseAddress       The physical address that is the start address of a memory region.\r
  @param[in]  Length            The size in bytes of the memory region.\r
  @param[in]  Attributes        The bit mask of attributes to modify for the memory region.\r
  @param[in]  PageAction        The page action.\r
  @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.\r
                                NULL mean page split is unsupported.\r
  @param[out] IsSplitted        TRUE means page table splitted. FALSE means page table not splitted.\r
  @param[out] IsModified        TRUE means page table modified. FALSE means page table not modified.\r
\r
  @retval RETURN_SUCCESS           The attributes were modified for the memory region.\r
  @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by\r
                                   BaseAddress and Length cannot be modified.\r
  @retval RETURN_INVALID_PARAMETER Length is zero.\r
                                   Attributes specified an illegal combination of attributes that\r
                                   cannot be set together.\r
  @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of\r
                                   the memory resource range.\r
  @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory\r
                                   resource range specified by BaseAddress and Length.\r
                                   The bit mask of attributes is not support for the memory resource\r
                                   range specified by BaseAddress and Length.\r
**/\r
RETURN_STATUS\r
ConvertMemoryPageAttributes (\r
  IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext OPTIONAL,\r
  IN  PHYSICAL_ADDRESS                  BaseAddress,\r
  IN  UINT64                            Length,\r
  IN  UINT64                            Attributes,\r
  IN  PAGE_ACTION                       PageAction,\r
  IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc OPTIONAL,\r
  OUT BOOLEAN                           *IsSplitted,  OPTIONAL\r
  OUT BOOLEAN                           *IsModified   OPTIONAL\r
  )\r
{\r
  PAGE_TABLE_LIB_PAGING_CONTEXT     CurrentPagingContext;\r
  UINT64                            *PageEntry;\r
  PAGE_ATTRIBUTE                    PageAttribute;\r
  UINTN                             PageEntryLength;\r
  PAGE_ATTRIBUTE                    SplitAttribute;\r
  RETURN_STATUS                     Status;\r
  BOOLEAN                           IsEntryModified;\r
  BOOLEAN                           IsWpEnabled;\r
\r
  if ((BaseAddress & (SIZE_4KB - 1)) != 0) {\r
    DEBUG ((DEBUG_ERROR, "BaseAddress(0x%lx) is not aligned!\n", BaseAddress));\r
    return EFI_UNSUPPORTED;\r
  }\r
  if ((Length & (SIZE_4KB - 1)) != 0) {\r
    DEBUG ((DEBUG_ERROR, "Length(0x%lx) is not aligned!\n", Length));\r
    return EFI_UNSUPPORTED;\r
  }\r
  if (Length == 0) {\r
    DEBUG ((DEBUG_ERROR, "Length is 0!\n"));\r
    return RETURN_INVALID_PARAMETER;\r
  }\r
\r
  if ((Attributes & ~(EFI_MEMORY_RP | EFI_MEMORY_RO | EFI_MEMORY_XP)) != 0) {\r
    DEBUG ((DEBUG_ERROR, "Attributes(0x%lx) has unsupported bit\n", Attributes));\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  if (PagingContext == NULL) {\r
    GetCurrentPagingContext (&CurrentPagingContext);\r
  } else {\r
    CopyMem (&CurrentPagingContext, PagingContext, sizeof(CurrentPagingContext));\r
  }\r
  switch(CurrentPagingContext.MachineType) {\r
  case IMAGE_FILE_MACHINE_I386:\r
    if (CurrentPagingContext.ContextData.Ia32.PageTableBase == 0) {\r
      if (Attributes == 0) {\r
        return EFI_SUCCESS;\r
      } else {\r
        DEBUG ((DEBUG_ERROR, "PageTable is 0!\n"));\r
        return EFI_UNSUPPORTED;\r
      }\r
    }\r
    if ((CurrentPagingContext.ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) == 0) {\r
      DEBUG ((DEBUG_ERROR, "Non-PAE Paging!\n"));\r
      return EFI_UNSUPPORTED;\r
    }\r
    if ((BaseAddress + Length) > BASE_4GB) {\r
      DEBUG ((DEBUG_ERROR, "Beyond 4GB memory in 32-bit mode!\n"));\r
      return EFI_UNSUPPORTED;\r
    }\r
    break;\r
  case IMAGE_FILE_MACHINE_X64:\r
    ASSERT (CurrentPagingContext.ContextData.X64.PageTableBase != 0);\r
    break;\r
  default:\r
    ASSERT(FALSE);\r
    return EFI_UNSUPPORTED;\r
    break;\r
  }\r
\r
//  DEBUG ((DEBUG_ERROR, "ConvertMemoryPageAttributes(%x) - %016lx, %016lx, %02lx\n", IsSet, BaseAddress, Length, Attributes));\r
\r
  if (IsSplitted != NULL) {\r
    *IsSplitted = FALSE;\r
  }\r
  if (IsModified != NULL) {\r
    *IsModified = FALSE;\r
  }\r
  if (AllocatePagesFunc == NULL) {\r
    AllocatePagesFunc = AllocatePageTableMemory;\r
  }\r
\r
  //\r
  // Make sure that the page table is changeable.\r
  //\r
  IsWpEnabled = IsReadOnlyPageWriteProtected ();\r
  if (IsWpEnabled) {\r
    DisableReadOnlyPageWriteProtect ();\r
  }\r
\r
  //\r
  // Below logic is to check 2M/4K page to make sure we donot waist memory.\r
  //\r
  Status = EFI_SUCCESS;\r
  while (Length != 0) {\r
    PageEntry = GetPageTableEntry (&CurrentPagingContext, BaseAddress, &PageAttribute);\r
    if (PageEntry == NULL) {\r
      Status = RETURN_UNSUPPORTED;\r
      goto Done;\r
    }\r
    PageEntryLength = PageAttributeToLength (PageAttribute);\r
    SplitAttribute = NeedSplitPage (BaseAddress, Length, PageEntry, PageAttribute);\r
    if (SplitAttribute == PageNone) {\r
      ConvertPageEntryAttribute (&CurrentPagingContext, PageEntry, Attributes, PageAction, &IsEntryModified);\r
      if (IsEntryModified) {\r
        if (IsModified != NULL) {\r
          *IsModified = TRUE;\r
        }\r
      }\r
      //\r
      // Convert success, move to next\r
      //\r
      BaseAddress += PageEntryLength;\r
      Length -= PageEntryLength;\r
    } else {\r
      if (AllocatePagesFunc == NULL) {\r
        Status = RETURN_UNSUPPORTED;\r
        goto Done;\r
      }\r
      Status = SplitPage (PageEntry, PageAttribute, SplitAttribute, AllocatePagesFunc);\r
      if (RETURN_ERROR (Status)) {\r
        Status = RETURN_UNSUPPORTED;\r
        goto Done;\r
      }\r
      if (IsSplitted != NULL) {\r
        *IsSplitted = TRUE;\r
      }\r
      if (IsModified != NULL) {\r
        *IsModified = TRUE;\r
      }\r
      //\r
      // Just split current page\r
      // Convert success in next around\r
      //\r
    }\r
  }\r
\r
Done:\r
  //\r
  // Restore page table write protection, if any.\r
  //\r
  if (IsWpEnabled) {\r
    EnableReadOnlyPageWriteProtect ();\r
  }\r
  return Status;\r
}\r
\r
/**\r
  This function assigns the page attributes for the memory region specified by BaseAddress and\r
  Length from their current attributes to the attributes specified by Attributes.\r
\r
  Caller should make sure BaseAddress and Length is at page boundary.\r
\r
  Caller need guarentee the TPL <= TPL_NOTIFY, if there is split page request.\r
\r
  @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.\r
  @param[in]  BaseAddress       The physical address that is the start address of a memory region.\r
  @param[in]  Length            The size in bytes of the memory region.\r
  @param[in]  Attributes        The bit mask of attributes to set for the memory region.\r
  @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.\r
                                NULL mean page split is unsupported.\r
\r
  @retval RETURN_SUCCESS           The attributes were cleared for the memory region.\r
  @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by\r
                                   BaseAddress and Length cannot be modified.\r
  @retval RETURN_INVALID_PARAMETER Length is zero.\r
                                   Attributes specified an illegal combination of attributes that\r
                                   cannot be set together.\r
  @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of\r
                                   the memory resource range.\r
  @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory\r
                                   resource range specified by BaseAddress and Length.\r
                                   The bit mask of attributes is not support for the memory resource\r
                                   range specified by BaseAddress and Length.\r
**/\r
RETURN_STATUS\r
EFIAPI\r
AssignMemoryPageAttributes (\r
  IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext OPTIONAL,\r
  IN  PHYSICAL_ADDRESS                  BaseAddress,\r
  IN  UINT64                            Length,\r
  IN  UINT64                            Attributes,\r
  IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc OPTIONAL\r
  )\r
{\r
  RETURN_STATUS  Status;\r
  BOOLEAN        IsModified;\r
  BOOLEAN        IsSplitted;\r
\r
//  DEBUG((DEBUG_INFO, "AssignMemoryPageAttributes: 0x%lx - 0x%lx (0x%lx)\n", BaseAddress, Length, Attributes));\r
  Status = ConvertMemoryPageAttributes (PagingContext, BaseAddress, Length, Attributes, PageActionAssign, AllocatePagesFunc, &IsSplitted, &IsModified);\r
  if (!EFI_ERROR(Status)) {\r
    if ((PagingContext == NULL) && IsModified) {\r
      //\r
      // Flush TLB as last step\r
      //\r
      CpuFlushTlb();\r
      SyncMemoryPageAttributesAp (SyncCpuFlushTlb);\r
    }\r
  }\r
\r
  return Status;\r
}\r
\r
/**\r
 Check if Execute Disable feature is enabled or not.\r
**/\r
BOOLEAN\r
IsExecuteDisableEnabled (\r
  VOID\r
  )\r
{\r
  MSR_CORE_IA32_EFER_REGISTER    MsrEfer;\r
\r
  MsrEfer.Uint64 = AsmReadMsr64 (MSR_IA32_EFER);\r
  return (MsrEfer.Bits.NXE == 1);\r
}\r
\r
/**\r
  Update GCD memory space attributes according to current page table setup.\r
**/\r
VOID\r
RefreshGcdMemoryAttributesFromPaging (\r
  VOID\r
  )\r
{\r
  EFI_STATUS                          Status;\r
  UINTN                               NumberOfDescriptors;\r
  EFI_GCD_MEMORY_SPACE_DESCRIPTOR     *MemorySpaceMap;\r
  PAGE_TABLE_LIB_PAGING_CONTEXT       PagingContext;\r
  PAGE_ATTRIBUTE                      PageAttribute;\r
  UINT64                              *PageEntry;\r
  UINT64                              PageLength;\r
  UINT64                              MemorySpaceLength;\r
  UINT64                              Length;\r
  UINT64                              BaseAddress;\r
  UINT64                              PageStartAddress;\r
  UINT64                              Attributes;\r
  UINT64                              Capabilities;\r
  UINT64                              NewAttributes;\r
  UINTN                               Index;\r
\r
  //\r
  // Assuming that memory space map returned is sorted already; otherwise sort\r
  // them in the order of lowest address to highest address.\r
  //\r
  Status = gDS->GetMemorySpaceMap (&NumberOfDescriptors, &MemorySpaceMap);\r
  ASSERT_EFI_ERROR (Status);\r
\r
  GetCurrentPagingContext (&PagingContext);\r
\r
  Attributes      = 0;\r
  NewAttributes   = 0;\r
  BaseAddress     = 0;\r
  PageLength      = 0;\r
\r
  if (IsExecuteDisableEnabled ()) {\r
    Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP | EFI_MEMORY_XP;\r
  } else {\r
    Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP;\r
  }\r
\r
  for (Index = 0; Index < NumberOfDescriptors; Index++) {\r
    if (MemorySpaceMap[Index].GcdMemoryType == EfiGcdMemoryTypeNonExistent) {\r
      continue;\r
    }\r
\r
    //\r
    // Sync the actual paging related capabilities back to GCD service first.\r
    // As a side effect (good one), this can also help to avoid unnecessary\r
    // memory map entries due to the different capabilities of the same type\r
    // memory, such as multiple RT_CODE and RT_DATA entries in memory map,\r
    // which could cause boot failure of some old Linux distro (before v4.3).\r
    //\r
    Status = gDS->SetMemorySpaceCapabilities (\r
                    MemorySpaceMap[Index].BaseAddress,\r
                    MemorySpaceMap[Index].Length,\r
                    MemorySpaceMap[Index].Capabilities | Capabilities\r
                    );\r
    if (EFI_ERROR (Status)) {\r
      //\r
      // If we cannot udpate the capabilities, we cannot update its\r
      // attributes either. So just simply skip current block of memory.\r
      //\r
      DEBUG ((\r
        DEBUG_WARN,\r
        "Failed to update capability: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",\r
        (UINT64)Index, MemorySpaceMap[Index].BaseAddress,\r
        MemorySpaceMap[Index].BaseAddress + MemorySpaceMap[Index].Length - 1,\r
        MemorySpaceMap[Index].Capabilities,\r
        MemorySpaceMap[Index].Capabilities | Capabilities\r
        ));\r
      continue;\r
    }\r
\r
    if (MemorySpaceMap[Index].BaseAddress >= (BaseAddress + PageLength)) {\r
      //\r
      // Current memory space starts at a new page. Resetting PageLength will\r
      // trigger a retrieval of page attributes at new address.\r
      //\r
      PageLength = 0;\r
    } else {\r
      //\r
      // In case current memory space is not adjacent to last one\r
      //\r
      PageLength -= (MemorySpaceMap[Index].BaseAddress - BaseAddress);\r
    }\r
\r
    //\r
    // Sync actual page attributes to GCD\r
    //\r
    BaseAddress       = MemorySpaceMap[Index].BaseAddress;\r
    MemorySpaceLength = MemorySpaceMap[Index].Length;\r
    while (MemorySpaceLength > 0) {\r
      if (PageLength == 0) {\r
        PageEntry = GetPageTableEntry (&PagingContext, BaseAddress, &PageAttribute);\r
        if (PageEntry == NULL) {\r
          break;\r
        }\r
\r
        //\r
        // Note current memory space might start in the middle of a page\r
        //\r
        PageStartAddress  = (*PageEntry) & (UINT64)PageAttributeToMask(PageAttribute);\r
        PageLength        = PageAttributeToLength (PageAttribute) - (BaseAddress - PageStartAddress);\r
        Attributes        = GetAttributesFromPageEntry (PageEntry);\r
      }\r
\r
      Length = MIN (PageLength, MemorySpaceLength);\r
      if (Attributes != (MemorySpaceMap[Index].Attributes &\r
                         EFI_MEMORY_PAGETYPE_MASK)) {\r
        NewAttributes = (MemorySpaceMap[Index].Attributes &\r
                         ~EFI_MEMORY_PAGETYPE_MASK) | Attributes;\r
        Status = gDS->SetMemorySpaceAttributes (\r
                        BaseAddress,\r
                        Length,\r
                        NewAttributes\r
                        );\r
        ASSERT_EFI_ERROR (Status);\r
        DEBUG ((\r
          DEBUG_VERBOSE,\r
          "Updated memory space attribute: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",\r
          (UINT64)Index, BaseAddress, BaseAddress + Length - 1,\r
          MemorySpaceMap[Index].Attributes,\r
          NewAttributes\r
          ));\r
      }\r
\r
      PageLength        -= Length;\r
      MemorySpaceLength -= Length;\r
      BaseAddress       += Length;\r
    }\r
  }\r
\r
  FreePool (MemorySpaceMap);\r
}\r
\r
/**\r
  Initialize a buffer pool for page table use only.\r
\r
  To reduce the potential split operation on page table, the pages reserved for\r
  page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and\r
  at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always\r
  initialized with number of pages greater than or equal to the given PoolPages.\r
\r
  Once the pages in the pool are used up, this method should be called again to\r
  reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. Usually this won't happen\r
  often in practice.\r
\r
  @param[in] PoolPages      The least page number of the pool to be created.\r
\r
  @retval TRUE    The pool is initialized successfully.\r
  @retval FALSE   The memory is out of resource.\r
**/\r
BOOLEAN\r
InitializePageTablePool (\r
  IN  UINTN                           PoolPages\r
  )\r
{\r
  VOID                      *Buffer;\r
  BOOLEAN                   IsModified;\r
\r
  //\r
  // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for\r
  // header.\r
  //\r
  PoolPages += 1;   // Add one page for header.\r
  PoolPages = ((PoolPages - 1) / PAGE_TABLE_POOL_UNIT_PAGES + 1) *\r
              PAGE_TABLE_POOL_UNIT_PAGES;\r
  Buffer = AllocateAlignedPages (PoolPages, PAGE_TABLE_POOL_ALIGNMENT);\r
  if (Buffer == NULL) {\r
    DEBUG ((DEBUG_ERROR, "ERROR: Out of aligned pages\r\n"));\r
    return FALSE;\r
  }\r
\r
  //\r
  // Link all pools into a list for easier track later.\r
  //\r
  if (mPageTablePool == NULL) {\r
    mPageTablePool = Buffer;\r
    mPageTablePool->NextPool = mPageTablePool;\r
  } else {\r
    ((PAGE_TABLE_POOL *)Buffer)->NextPool = mPageTablePool->NextPool;\r
    mPageTablePool->NextPool = Buffer;\r
    mPageTablePool = Buffer;\r
  }\r
\r
  //\r
  // Reserve one page for pool header.\r
  //\r
  mPageTablePool->FreePages  = PoolPages - 1;\r
  mPageTablePool->Offset = EFI_PAGES_TO_SIZE (1);\r
\r
  //\r
  // Mark the whole pool pages as read-only.\r
  //\r
  ConvertMemoryPageAttributes (\r
    NULL,\r
    (PHYSICAL_ADDRESS)(UINTN)Buffer,\r
    EFI_PAGES_TO_SIZE (PoolPages),\r
    EFI_MEMORY_RO,\r
    PageActionSet,\r
    AllocatePageTableMemory,\r
    NULL,\r
    &IsModified\r
    );\r
  ASSERT (IsModified == TRUE);\r
\r
  return TRUE;\r
}\r
\r
/**\r
  This API provides a way to allocate memory for page table.\r
\r
  This API can be called more than once to allocate memory for page tables.\r
\r
  Allocates the number of 4KB pages and returns a pointer to the allocated\r
  buffer. The buffer returned is aligned on a 4KB boundary.\r
\r
  If Pages is 0, then NULL is returned.\r
  If there is not enough memory remaining to satisfy the request, then NULL is\r
  returned.\r
\r
  @param  Pages                 The number of 4 KB pages to allocate.\r
\r
  @return A pointer to the allocated buffer or NULL if allocation fails.\r
\r
**/\r
VOID *\r
EFIAPI\r
AllocatePageTableMemory (\r
  IN UINTN           Pages\r
  )\r
{\r
  VOID                            *Buffer;\r
\r
  if (Pages == 0) {\r
    return NULL;\r
  }\r
\r
  //\r
  // Renew the pool if necessary.\r
  //\r
  if (mPageTablePool == NULL ||\r
      Pages > mPageTablePool->FreePages) {\r
    if (!InitializePageTablePool (Pages)) {\r
      return NULL;\r
    }\r
  }\r
\r
  Buffer = (UINT8 *)mPageTablePool + mPageTablePool->Offset;\r
\r
  mPageTablePool->Offset     += EFI_PAGES_TO_SIZE (Pages);\r
  mPageTablePool->FreePages  -= Pages;\r
\r
  return Buffer;\r
}\r
\r
/**\r
  Initialize the Page Table lib.\r
**/\r
VOID\r
InitializePageTableLib (\r
  VOID\r
  )\r
{\r
  PAGE_TABLE_LIB_PAGING_CONTEXT     CurrentPagingContext;\r
\r
  GetCurrentPagingContext (&CurrentPagingContext);\r
\r
  //\r
  // Reserve memory of page tables for future uses, if paging is enabled.\r
  //\r
  if (CurrentPagingContext.ContextData.X64.PageTableBase != 0 &&\r
      (CurrentPagingContext.ContextData.Ia32.Attributes &\r
       PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0) {\r
    DisableReadOnlyPageWriteProtect ();\r
    InitializePageTablePool (1);\r
    EnableReadOnlyPageWriteProtect ();\r
  }\r
\r
  DEBUG ((DEBUG_INFO, "CurrentPagingContext:\n", CurrentPagingContext.MachineType));\r
  DEBUG ((DEBUG_INFO, "  MachineType   - 0x%x\n", CurrentPagingContext.MachineType));\r
  DEBUG ((DEBUG_INFO, "  PageTableBase - 0x%x\n", CurrentPagingContext.ContextData.X64.PageTableBase));\r
  DEBUG ((DEBUG_INFO, "  Attributes    - 0x%x\n", CurrentPagingContext.ContextData.X64.Attributes));\r
\r
  return ;\r
}\r
\r