[mirror_edk2.git] / ArmPkg / Library / ArmMmuLib / AArch64 / ArmMmuLibCore.c

/** @file\r
*  File managing the MMU for ARMv8 architecture\r
*\r
*  Copyright (c) 2011-2020, ARM Limited. All rights reserved.\r
*  Copyright (c) 2016, Linaro Limited. All rights reserved.\r
*  Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>\r
*\r
*  SPDX-License-Identifier: BSD-2-Clause-Patent\r
*\r
**/\r
\r
#include <Uefi.h>\r
#include <Chipset/AArch64.h>\r
#include <Library/BaseMemoryLib.h>\r
#include <Library/CacheMaintenanceLib.h>\r
#include <Library/MemoryAllocationLib.h>\r
#include <Library/ArmLib.h>\r
#include <Library/ArmMmuLib.h>\r
#include <Library/BaseLib.h>\r
#include <Library/DebugLib.h>\r
#include <Library/HobLib.h>\r
\r
STATIC\r
VOID (\r
  EFIAPI  *mReplaceLiveEntryFunc\r
  )(\r
    IN  UINT64  *Entry,\r
    IN  UINT64  Value,\r
    IN  UINT64  RegionStart,\r
    IN  BOOLEAN DisableMmu\r
    ) = ArmReplaceLiveTranslationEntry;\r
\r
STATIC\r
UINT64\r
ArmMemoryAttributeToPageAttribute (\r
  IN ARM_MEMORY_REGION_ATTRIBUTES  Attributes\r
  )\r
{\r
  switch (Attributes) {\r
    case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK_NONSHAREABLE:\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK_NONSHAREABLE:\r
      return TT_ATTR_INDX_MEMORY_WRITE_BACK;\r
\r
    case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK:\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK:\r
      return TT_ATTR_INDX_MEMORY_WRITE_BACK | TT_SH_INNER_SHAREABLE;\r
\r
    case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_THROUGH:\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_THROUGH:\r
      return TT_ATTR_INDX_MEMORY_WRITE_THROUGH | TT_SH_INNER_SHAREABLE;\r
\r
    // Uncached and device mappings are treated as outer shareable by default,\r
    case ARM_MEMORY_REGION_ATTRIBUTE_UNCACHED_UNBUFFERED:\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_UNCACHED_UNBUFFERED:\r
      return TT_ATTR_INDX_MEMORY_NON_CACHEABLE;\r
\r
    default:\r
      ASSERT (0);\r
    case ARM_MEMORY_REGION_ATTRIBUTE_DEVICE:\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_DEVICE:\r
      if (ArmReadCurrentEL () == AARCH64_EL2) {\r
        return TT_ATTR_INDX_DEVICE_MEMORY | TT_XN_MASK;\r
      } else {\r
        return TT_ATTR_INDX_DEVICE_MEMORY | TT_UXN_MASK | TT_PXN_MASK;\r
      }\r
  }\r
}\r
\r
#define MIN_T0SZ        16\r
#define BITS_PER_LEVEL  9\r
#define MAX_VA_BITS     48\r
\r
STATIC\r
UINTN\r
GetRootTableEntryCount (\r
  IN  UINTN  T0SZ\r
  )\r
{\r
  return TT_ENTRY_COUNT >> (T0SZ - MIN_T0SZ) % BITS_PER_LEVEL;\r
}\r
\r
STATIC\r
UINTN\r
GetRootTableLevel (\r
  IN  UINTN  T0SZ\r
  )\r
{\r
  return (T0SZ - MIN_T0SZ) / BITS_PER_LEVEL;\r
}\r
\r
STATIC\r
VOID\r
ReplaceTableEntry (\r
  IN  UINT64   *Entry,\r
  IN  UINT64   Value,\r
  IN  UINT64   RegionStart,\r
  IN  UINT64   BlockMask,\r
  IN  BOOLEAN  IsLiveBlockMapping\r
  )\r
{\r
  BOOLEAN  DisableMmu;\r
\r
  //\r
  // Replacing a live block entry with a table entry (or vice versa) requires a\r
  // break-before-make sequence as per the architecture. This means the mapping\r
  // must be made invalid and cleaned from the TLBs first, and this is a bit of\r
  // a hassle if the mapping in question covers the code that is actually doing\r
  // the mapping and the unmapping, and so we only bother with this if actually\r
  // necessary.\r
  //\r
\r
  if (!IsLiveBlockMapping || !ArmMmuEnabled ()) {\r
    // If the mapping is not a live block mapping, or the MMU is not on yet, we\r
    // can simply overwrite the entry.\r
    *Entry = Value;\r
    ArmUpdateTranslationTableEntry (Entry, (VOID *)(UINTN)RegionStart);\r
  } else {\r
    // If the mapping in question does not cover the code that updates the\r
    // entry in memory, or the entry that we are intending to update, we can\r
    // use an ordinary break before make. Otherwise, we will need to\r
    // temporarily disable the MMU.\r
    DisableMmu = FALSE;\r
    if ((((RegionStart ^ (UINTN)ArmReplaceLiveTranslationEntry) & ~BlockMask) == 0) ||\r
        (((RegionStart ^ (UINTN)Entry) & ~BlockMask) == 0))\r
    {\r
      DisableMmu = TRUE;\r
      DEBUG ((DEBUG_WARN, "%a: splitting block entry with MMU disabled\n", __FUNCTION__));\r
    }\r
\r
    ArmReplaceLiveTranslationEntry (Entry, Value, RegionStart, DisableMmu);\r
  }\r
}\r
\r
STATIC\r
VOID\r
FreePageTablesRecursive (\r
  IN  UINT64  *TranslationTable,\r
  IN  UINTN   Level\r
  )\r
{\r
  UINTN  Index;\r
\r
  ASSERT (Level <= 3);\r
\r
  if (Level < 3) {\r
    for (Index = 0; Index < TT_ENTRY_COUNT; Index++) {\r
      if ((TranslationTable[Index] & TT_TYPE_MASK) == TT_TYPE_TABLE_ENTRY) {\r
        FreePageTablesRecursive (\r
          (VOID *)(UINTN)(TranslationTable[Index] &\r
                          TT_ADDRESS_MASK_BLOCK_ENTRY),\r
          Level + 1\r
          );\r
      }\r
    }\r
  }\r
\r
  FreePages (TranslationTable, 1);\r
}\r
\r
STATIC\r
BOOLEAN\r
IsBlockEntry (\r
  IN  UINT64  Entry,\r
  IN  UINTN   Level\r
  )\r
{\r
  if (Level == 3) {\r
    return (Entry & TT_TYPE_MASK) == TT_TYPE_BLOCK_ENTRY_LEVEL3;\r
  }\r
\r
  return (Entry & TT_TYPE_MASK) == TT_TYPE_BLOCK_ENTRY;\r
}\r
\r
STATIC\r
BOOLEAN\r
IsTableEntry (\r
  IN  UINT64  Entry,\r
  IN  UINTN   Level\r
  )\r
{\r
  if (Level == 3) {\r
    //\r
    // TT_TYPE_TABLE_ENTRY aliases TT_TYPE_BLOCK_ENTRY_LEVEL3\r
    // so we need to take the level into account as well.\r
    //\r
    return FALSE;\r
  }\r
\r
  return (Entry & TT_TYPE_MASK) == TT_TYPE_TABLE_ENTRY;\r
}\r
\r
STATIC\r
EFI_STATUS\r
UpdateRegionMappingRecursive (\r
  IN  UINT64   RegionStart,\r
  IN  UINT64   RegionEnd,\r
  IN  UINT64   AttributeSetMask,\r
  IN  UINT64   AttributeClearMask,\r
  IN  UINT64   *PageTable,\r
  IN  UINTN    Level,\r
  IN  BOOLEAN  TableIsLive\r
  )\r
{\r
  UINTN       BlockShift;\r
  UINT64      BlockMask;\r
  UINT64      BlockEnd;\r
  UINT64      *Entry;\r
  UINT64      EntryValue;\r
  VOID        *TranslationTable;\r
  EFI_STATUS  Status;\r
  BOOLEAN     NextTableIsLive;\r
\r
  ASSERT (((RegionStart | RegionEnd) & EFI_PAGE_MASK) == 0);\r
\r
  BlockShift = (Level + 1) * BITS_PER_LEVEL + MIN_T0SZ;\r
  BlockMask  = MAX_UINT64 >> BlockShift;\r
\r
  DEBUG ((\r
    DEBUG_VERBOSE,\r
    "%a(%d): %llx - %llx set %lx clr %lx\n",\r
    __FUNCTION__,\r
    Level,\r
    RegionStart,\r
    RegionEnd,\r
    AttributeSetMask,\r
    AttributeClearMask\r
    ));\r
\r
  for ( ; RegionStart < RegionEnd; RegionStart = BlockEnd) {\r
    BlockEnd = MIN (RegionEnd, (RegionStart | BlockMask) + 1);\r
    Entry    = &PageTable[(RegionStart >> (64 - BlockShift)) & (TT_ENTRY_COUNT - 1)];\r
\r
    //\r
    // If RegionStart or BlockEnd is not aligned to the block size at this\r
    // level, we will have to create a table mapping in order to map less\r
    // than a block, and recurse to create the block or page entries at\r
    // the next level. No block mappings are allowed at all at level 0,\r
    // so in that case, we have to recurse unconditionally.\r
    //\r
    // One special case to take into account is any region that covers the page\r
    // table itself: if we'd cover such a region with block mappings, we are\r
    // more likely to end up in the situation later where we need to disable\r
    // the MMU in order to update page table entries safely, so prefer page\r
    // mappings in that particular case.\r
    //\r
    if ((Level == 0) || (((RegionStart | BlockEnd) & BlockMask) != 0) ||\r
        ((Level < 3) && (((UINT64)PageTable & ~BlockMask) == RegionStart)) ||\r
        IsTableEntry (*Entry, Level))\r
    {\r
      ASSERT (Level < 3);\r
\r
      if (!IsTableEntry (*Entry, Level)) {\r
        //\r
        // No table entry exists yet, so we need to allocate a page table\r
        // for the next level.\r
        //\r
        TranslationTable = AllocatePages (1);\r
        if (TranslationTable == NULL) {\r
          return EFI_OUT_OF_RESOURCES;\r
        }\r
\r
        if (!ArmMmuEnabled ()) {\r
          //\r
          // Make sure we are not inadvertently hitting in the caches\r
          // when populating the page tables.\r
          //\r
          InvalidateDataCacheRange (TranslationTable, EFI_PAGE_SIZE);\r
        }\r
\r
        ZeroMem (TranslationTable, EFI_PAGE_SIZE);\r
\r
        if (IsBlockEntry (*Entry, Level)) {\r
          //\r
          // We are splitting an existing block entry, so we have to populate\r
          // the new table with the attributes of the block entry it replaces.\r
          //\r
          Status = UpdateRegionMappingRecursive (\r
                     RegionStart & ~BlockMask,\r
                     (RegionStart | BlockMask) + 1,\r
                     *Entry & TT_ATTRIBUTES_MASK,\r
                     0,\r
                     TranslationTable,\r
                     Level + 1,\r
                     FALSE\r
                     );\r
          if (EFI_ERROR (Status)) {\r
            //\r
            // The range we passed to UpdateRegionMappingRecursive () is block\r
            // aligned, so it is guaranteed that no further pages were allocated\r
            // by it, and so we only have to free the page we allocated here.\r
            //\r
            FreePages (TranslationTable, 1);\r
            return Status;\r
          }\r
        }\r
\r
        NextTableIsLive = FALSE;\r
      } else {\r
        TranslationTable = (VOID *)(UINTN)(*Entry & TT_ADDRESS_MASK_BLOCK_ENTRY);\r
        NextTableIsLive  = TableIsLive;\r
      }\r
\r
      //\r
      // Recurse to the next level\r
      //\r
      Status = UpdateRegionMappingRecursive (\r
                 RegionStart,\r
                 BlockEnd,\r
                 AttributeSetMask,\r
                 AttributeClearMask,\r
                 TranslationTable,\r
                 Level + 1,\r
                 NextTableIsLive\r
                 );\r
      if (EFI_ERROR (Status)) {\r
        if (!IsTableEntry (*Entry, Level)) {\r
          //\r
          // We are creating a new table entry, so on failure, we can free all\r
          // allocations we made recursively, given that the whole subhierarchy\r
          // has not been wired into the live page tables yet. (This is not\r
          // possible for existing table entries, since we cannot revert the\r
          // modifications we made to the subhierarchy it represents.)\r
          //\r
          FreePageTablesRecursive (TranslationTable, Level + 1);\r
        }\r
\r
        return Status;\r
      }\r
\r
      if (!IsTableEntry (*Entry, Level)) {\r
        EntryValue = (UINTN)TranslationTable | TT_TYPE_TABLE_ENTRY;\r
        ReplaceTableEntry (\r
          Entry,\r
          EntryValue,\r
          RegionStart,\r
          BlockMask,\r
          TableIsLive && IsBlockEntry (*Entry, Level)\r
          );\r
      }\r
    } else {\r
      EntryValue  = (*Entry & AttributeClearMask) | AttributeSetMask;\r
      EntryValue |= RegionStart;\r
      EntryValue |= (Level == 3) ? TT_TYPE_BLOCK_ENTRY_LEVEL3\r
                                 : TT_TYPE_BLOCK_ENTRY;\r
\r
      ReplaceTableEntry (Entry, EntryValue, RegionStart, BlockMask, FALSE);\r
    }\r
  }\r
\r
  return EFI_SUCCESS;\r
}\r
\r
STATIC\r
EFI_STATUS\r
UpdateRegionMapping (\r
  IN  UINT64   RegionStart,\r
  IN  UINT64   RegionLength,\r
  IN  UINT64   AttributeSetMask,\r
  IN  UINT64   AttributeClearMask,\r
  IN  BOOLEAN  TableIsLive\r
  )\r
{\r
  UINTN  T0SZ;\r
\r
  if (((RegionStart | RegionLength) & EFI_PAGE_MASK) != 0) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  T0SZ = ArmGetTCR () & TCR_T0SZ_MASK;\r
\r
  return UpdateRegionMappingRecursive (\r
           RegionStart,\r
           RegionStart + RegionLength,\r
           AttributeSetMask,\r
           AttributeClearMask,\r
           ArmGetTTBR0BaseAddress (),\r
           GetRootTableLevel (T0SZ),\r
           TableIsLive\r
           );\r
}\r
\r
STATIC\r
EFI_STATUS\r
FillTranslationTable (\r
  IN  UINT64                        *RootTable,\r
  IN  ARM_MEMORY_REGION_DESCRIPTOR  *MemoryRegion\r
  )\r
{\r
  return UpdateRegionMapping (\r
           MemoryRegion->VirtualBase,\r
           MemoryRegion->Length,\r
           ArmMemoryAttributeToPageAttribute (MemoryRegion->Attributes) | TT_AF,\r
           0,\r
           FALSE\r
           );\r
}\r
\r
STATIC\r
UINT64\r
GcdAttributeToPageAttribute (\r
  IN UINT64  GcdAttributes\r
  )\r
{\r
  UINT64  PageAttributes;\r
\r
  switch (GcdAttributes & EFI_MEMORY_CACHETYPE_MASK) {\r
    case EFI_MEMORY_UC:\r
      PageAttributes = TT_ATTR_INDX_DEVICE_MEMORY;\r
      break;\r
    case EFI_MEMORY_WC:\r
      PageAttributes = TT_ATTR_INDX_MEMORY_NON_CACHEABLE;\r
      break;\r
    case EFI_MEMORY_WT:\r
      PageAttributes = TT_ATTR_INDX_MEMORY_WRITE_THROUGH | TT_SH_INNER_SHAREABLE;\r
      break;\r
    case EFI_MEMORY_WB:\r
      PageAttributes = TT_ATTR_INDX_MEMORY_WRITE_BACK | TT_SH_INNER_SHAREABLE;\r
      break;\r
    default:\r
      PageAttributes = TT_ATTR_INDX_MASK;\r
      break;\r
  }\r
\r
  if (((GcdAttributes & EFI_MEMORY_XP) != 0) ||\r
      ((GcdAttributes & EFI_MEMORY_CACHETYPE_MASK) == EFI_MEMORY_UC))\r
  {\r
    if (ArmReadCurrentEL () == AARCH64_EL2) {\r
      PageAttributes |= TT_XN_MASK;\r
    } else {\r
      PageAttributes |= TT_UXN_MASK | TT_PXN_MASK;\r
    }\r
  }\r
\r
  if ((GcdAttributes & EFI_MEMORY_RO) != 0) {\r
    PageAttributes |= TT_AP_NO_RO;\r
  }\r
\r
  return PageAttributes | TT_AF;\r
}\r
\r
EFI_STATUS\r
ArmSetMemoryAttributes (\r
  IN EFI_PHYSICAL_ADDRESS  BaseAddress,\r
  IN UINT64                Length,\r
  IN UINT64                Attributes\r
  )\r
{\r
  UINT64  PageAttributes;\r
  UINT64  PageAttributeMask;\r
\r
  PageAttributes    = GcdAttributeToPageAttribute (Attributes);\r
  PageAttributeMask = 0;\r
\r
  if ((Attributes & EFI_MEMORY_CACHETYPE_MASK) == 0) {\r
    //\r
    // No memory type was set in Attributes, so we are going to update the\r
    // permissions only.\r
    //\r
    PageAttributes   &= TT_AP_MASK | TT_UXN_MASK | TT_PXN_MASK;\r
    PageAttributeMask = ~(TT_ADDRESS_MASK_BLOCK_ENTRY | TT_AP_MASK |\r
                          TT_PXN_MASK | TT_XN_MASK);\r
  }\r
\r
  return UpdateRegionMapping (\r
           BaseAddress,\r
           Length,\r
           PageAttributes,\r
           PageAttributeMask,\r
           TRUE\r
           );\r
}\r
\r
STATIC\r
EFI_STATUS\r
SetMemoryRegionAttribute (\r
  IN  EFI_PHYSICAL_ADDRESS  BaseAddress,\r
  IN  UINT64                Length,\r
  IN  UINT64                Attributes,\r
  IN  UINT64                BlockEntryMask\r
  )\r
{\r
  return UpdateRegionMapping (\r
           BaseAddress,\r
           Length,\r
           Attributes,\r
           BlockEntryMask,\r
           TRUE\r
           );\r
}\r
\r
EFI_STATUS\r
ArmSetMemoryRegionNoExec (\r
  IN  EFI_PHYSICAL_ADDRESS  BaseAddress,\r
  IN  UINT64                Length\r
  )\r
{\r
  UINT64  Val;\r
\r
  if (ArmReadCurrentEL () == AARCH64_EL1) {\r
    Val = TT_PXN_MASK | TT_UXN_MASK;\r
  } else {\r
    Val = TT_XN_MASK;\r
  }\r
\r
  return SetMemoryRegionAttribute (\r
           BaseAddress,\r
           Length,\r
           Val,\r
           ~TT_ADDRESS_MASK_BLOCK_ENTRY\r
           );\r
}\r
\r
EFI_STATUS\r
ArmClearMemoryRegionNoExec (\r
  IN  EFI_PHYSICAL_ADDRESS  BaseAddress,\r
  IN  UINT64                Length\r
  )\r
{\r
  UINT64  Mask;\r
\r
  // XN maps to UXN in the EL1&0 translation regime\r
  Mask = ~(TT_ADDRESS_MASK_BLOCK_ENTRY | TT_PXN_MASK | TT_XN_MASK);\r
\r
  return SetMemoryRegionAttribute (\r
           BaseAddress,\r
           Length,\r
           0,\r
           Mask\r
           );\r
}\r
\r
EFI_STATUS\r
ArmSetMemoryRegionReadOnly (\r
  IN  EFI_PHYSICAL_ADDRESS  BaseAddress,\r
  IN  UINT64                Length\r
  )\r
{\r
  return SetMemoryRegionAttribute (\r
           BaseAddress,\r
           Length,\r
           TT_AP_NO_RO,\r
           ~TT_ADDRESS_MASK_BLOCK_ENTRY\r
           );\r
}\r
\r
EFI_STATUS\r
ArmClearMemoryRegionReadOnly (\r
  IN  EFI_PHYSICAL_ADDRESS  BaseAddress,\r
  IN  UINT64                Length\r
  )\r
{\r
  return SetMemoryRegionAttribute (\r
           BaseAddress,\r
           Length,\r
           TT_AP_NO_RW,\r
           ~(TT_ADDRESS_MASK_BLOCK_ENTRY | TT_AP_MASK)\r
           );\r
}\r
\r
EFI_STATUS\r
EFIAPI\r
ArmConfigureMmu (\r
  IN  ARM_MEMORY_REGION_DESCRIPTOR  *MemoryTable,\r
  OUT VOID                          **TranslationTableBase OPTIONAL,\r
  OUT UINTN                         *TranslationTableSize OPTIONAL\r
  )\r
{\r
  VOID        *TranslationTable;\r
  UINTN       MaxAddressBits;\r
  UINT64      MaxAddress;\r
  UINTN       T0SZ;\r
  UINTN       RootTableEntryCount;\r
  UINT64      TCR;\r
  EFI_STATUS  Status;\r
\r
  if (MemoryTable == NULL) {\r
    ASSERT (MemoryTable != NULL);\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  //\r
  // Limit the virtual address space to what we can actually use: UEFI\r
  // mandates a 1:1 mapping, so no point in making the virtual address\r
  // space larger than the physical address space. We also have to take\r
  // into account the architectural limitations that result from UEFI's\r
  // use of 4 KB pages.\r
  //\r
  MaxAddressBits = MIN (ArmGetPhysicalAddressBits (), MAX_VA_BITS);\r
  MaxAddress     = LShiftU64 (1ULL, MaxAddressBits) - 1;\r
\r
  T0SZ                = 64 - MaxAddressBits;\r
  RootTableEntryCount = GetRootTableEntryCount (T0SZ);\r
\r
  //\r
  // Set TCR that allows us to retrieve T0SZ in the subsequent functions\r
  //\r
  // Ideally we will be running at EL2, but should support EL1 as well.\r
  // UEFI should not run at EL3.\r
  if (ArmReadCurrentEL () == AARCH64_EL2) {\r
    // Note: Bits 23 and 31 are reserved(RES1) bits in TCR_EL2\r
    TCR = T0SZ | (1UL << 31) | (1UL << 23) | TCR_TG0_4KB;\r
\r
    // Set the Physical Address Size using MaxAddress\r
    if (MaxAddress < SIZE_4GB) {\r
      TCR |= TCR_PS_4GB;\r
    } else if (MaxAddress < SIZE_64GB) {\r
      TCR |= TCR_PS_64GB;\r
    } else if (MaxAddress < SIZE_1TB) {\r
      TCR |= TCR_PS_1TB;\r
    } else if (MaxAddress < SIZE_4TB) {\r
      TCR |= TCR_PS_4TB;\r
    } else if (MaxAddress < SIZE_16TB) {\r
      TCR |= TCR_PS_16TB;\r
    } else if (MaxAddress < SIZE_256TB) {\r
      TCR |= TCR_PS_256TB;\r
    } else {\r
      DEBUG ((\r
        DEBUG_ERROR,\r
        "ArmConfigureMmu: The MaxAddress 0x%lX is not supported by this MMU configuration.\n",\r
        MaxAddress\r
        ));\r
      ASSERT (0); // Bigger than 48-bit memory space are not supported\r
      return EFI_UNSUPPORTED;\r
    }\r
  } else if (ArmReadCurrentEL () == AARCH64_EL1) {\r
    // Due to Cortex-A57 erratum #822227 we must set TG1[1] == 1, regardless of EPD1.\r
    TCR = T0SZ | TCR_TG0_4KB | TCR_TG1_4KB | TCR_EPD1;\r
\r
    // Set the Physical Address Size using MaxAddress\r
    if (MaxAddress < SIZE_4GB) {\r
      TCR |= TCR_IPS_4GB;\r
    } else if (MaxAddress < SIZE_64GB) {\r
      TCR |= TCR_IPS_64GB;\r
    } else if (MaxAddress < SIZE_1TB) {\r
      TCR |= TCR_IPS_1TB;\r
    } else if (MaxAddress < SIZE_4TB) {\r
      TCR |= TCR_IPS_4TB;\r
    } else if (MaxAddress < SIZE_16TB) {\r
      TCR |= TCR_IPS_16TB;\r
    } else if (MaxAddress < SIZE_256TB) {\r
      TCR |= TCR_IPS_256TB;\r
    } else {\r
      DEBUG ((\r
        DEBUG_ERROR,\r
        "ArmConfigureMmu: The MaxAddress 0x%lX is not supported by this MMU configuration.\n",\r
        MaxAddress\r
        ));\r
      ASSERT (0); // Bigger than 48-bit memory space are not supported\r
      return EFI_UNSUPPORTED;\r
    }\r
  } else {\r
    ASSERT (0); // UEFI is only expected to run at EL2 and EL1, not EL3.\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  //\r
  // Translation table walks are always cache coherent on ARMv8-A, so cache\r
  // maintenance on page tables is never needed. Since there is a risk of\r
  // loss of coherency when using mismatched attributes, and given that memory\r
  // is mapped cacheable except for extraordinary cases (such as non-coherent\r
  // DMA), have the page table walker perform cached accesses as well, and\r
  // assert below that matches the attributes we use for CPU accesses to\r
  // the region.\r
  //\r
  TCR |= TCR_SH_INNER_SHAREABLE |\r
         TCR_RGN_OUTER_WRITE_BACK_ALLOC |\r
         TCR_RGN_INNER_WRITE_BACK_ALLOC;\r
\r
  // Set TCR\r
  ArmSetTCR (TCR);\r
\r
  // Allocate pages for translation table\r
  TranslationTable = AllocatePages (1);\r
  if (TranslationTable == NULL) {\r
    return EFI_OUT_OF_RESOURCES;\r
  }\r
\r
  //\r
  // We set TTBR0 just after allocating the table to retrieve its location from\r
  // the subsequent functions without needing to pass this value across the\r
  // functions. The MMU is only enabled after the translation tables are\r
  // populated.\r
  //\r
  ArmSetTTBR0 (TranslationTable);\r
\r
  if (TranslationTableBase != NULL) {\r
    *TranslationTableBase = TranslationTable;\r
  }\r
\r
  if (TranslationTableSize != NULL) {\r
    *TranslationTableSize = RootTableEntryCount * sizeof (UINT64);\r
  }\r
\r
  //\r
  // Make sure we are not inadvertently hitting in the caches\r
  // when populating the page tables.\r
  //\r
  InvalidateDataCacheRange (\r
    TranslationTable,\r
    RootTableEntryCount * sizeof (UINT64)\r
    );\r
  ZeroMem (TranslationTable, RootTableEntryCount * sizeof (UINT64));\r
\r
  while (MemoryTable->Length != 0) {\r
    Status = FillTranslationTable (TranslationTable, MemoryTable);\r
    if (EFI_ERROR (Status)) {\r
      goto FreeTranslationTable;\r
    }\r
\r
    MemoryTable++;\r
  }\r
\r
  //\r
  // EFI_MEMORY_UC ==> MAIR_ATTR_DEVICE_MEMORY\r
  // EFI_MEMORY_WC ==> MAIR_ATTR_NORMAL_MEMORY_NON_CACHEABLE\r
  // EFI_MEMORY_WT ==> MAIR_ATTR_NORMAL_MEMORY_WRITE_THROUGH\r
  // EFI_MEMORY_WB ==> MAIR_ATTR_NORMAL_MEMORY_WRITE_BACK\r
  //\r
  ArmSetMAIR (\r
    MAIR_ATTR (TT_ATTR_INDX_DEVICE_MEMORY, MAIR_ATTR_DEVICE_MEMORY)               |\r
    MAIR_ATTR (TT_ATTR_INDX_MEMORY_NON_CACHEABLE, MAIR_ATTR_NORMAL_MEMORY_NON_CACHEABLE) |\r
    MAIR_ATTR (TT_ATTR_INDX_MEMORY_WRITE_THROUGH, MAIR_ATTR_NORMAL_MEMORY_WRITE_THROUGH) |\r
    MAIR_ATTR (TT_ATTR_INDX_MEMORY_WRITE_BACK, MAIR_ATTR_NORMAL_MEMORY_WRITE_BACK)\r
    );\r
\r
  ArmDisableAlignmentCheck ();\r
  ArmEnableStackAlignmentCheck ();\r
  ArmEnableInstructionCache ();\r
  ArmEnableDataCache ();\r
\r
  ArmEnableMmu ();\r
  return EFI_SUCCESS;\r
\r
FreeTranslationTable:\r
  FreePages (TranslationTable, 1);\r
  return Status;\r
}\r
\r
RETURN_STATUS\r
EFIAPI\r
ArmMmuBaseLibConstructor (\r
  VOID\r
  )\r
{\r
  extern UINT32  ArmReplaceLiveTranslationEntrySize;\r
\r
  //\r
  // The ArmReplaceLiveTranslationEntry () helper function may be invoked\r
  // with the MMU off so we have to ensure that it gets cleaned to the PoC\r
  //\r
  WriteBackDataCacheRange (\r
    (VOID *)(UINTN)ArmReplaceLiveTranslationEntry,\r
    ArmReplaceLiveTranslationEntrySize\r
    );\r
\r
  return RETURN_SUCCESS;\r
}\r