X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=ArmPkg%2FLibrary%2FArmMmuLib%2FAArch64%2FArmMmuLibCore.c;h=d16e847218b7495c871675e289a561485ac02241;hb=49188b2aa481e76190883e73147af72128c878db;hp=6e05e6085011a4bbbc8dd45d03173bc69d001532;hpb=d7f03464b246b4111c8e21ba3dcd7b4a7b85fa7f;p=mirror_edk2.git diff --git a/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c b/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c index 6e05e60850..d16e847218 100644 --- a/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c +++ b/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c @@ -1,16 +1,11 @@ /** @file * File managing the MMU for ARMv8 architecture * -* Copyright (c) 2011-2014, ARM Limited. All rights reserved. +* Copyright (c) 2011-2020, ARM Limited. All rights reserved. * Copyright (c) 2016, Linaro Limited. All rights reserved. +* Copyright (c) 2017, Intel Corporation. All rights reserved.
* -* This program and the accompanying materials -* are licensed and made available under the terms and conditions of the BSD License -* which accompanies this distribution. The full text of the license may be found at -* http://opensource.org/licenses/bsd-license.php -* -* THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, -* WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. +* SPDX-License-Identifier: BSD-2-Clause-Patent * **/ @@ -34,6 +29,10 @@ ArmMemoryAttributeToPageAttribute ( ) { switch (Attributes) { + case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK_NONSHAREABLE: + case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK_NONSHAREABLE: + return TT_ATTR_INDX_MEMORY_WRITE_BACK; + case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK: case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK: return TT_ATTR_INDX_MEMORY_WRITE_BACK | TT_SH_INNER_SHAREABLE; @@ -48,7 +47,7 @@ ArmMemoryAttributeToPageAttribute ( return TT_ATTR_INDX_MEMORY_NON_CACHEABLE; default: - ASSERT(0); + ASSERT (0); case ARM_MEMORY_REGION_ATTRIBUTE_DEVICE: case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_DEVICE: if (ArmReadCurrentEL () == AARCH64_EL2) @@ -58,83 +57,8 @@ ArmMemoryAttributeToPageAttribute ( } } -UINT64 -PageAttributeToGcdAttribute ( - IN UINT64 PageAttributes - ) -{ - UINT64 GcdAttributes; - - switch (PageAttributes & TT_ATTR_INDX_MASK) { - case TT_ATTR_INDX_DEVICE_MEMORY: - GcdAttributes = EFI_MEMORY_UC; - break; - case TT_ATTR_INDX_MEMORY_NON_CACHEABLE: - GcdAttributes = EFI_MEMORY_WC; - break; - case TT_ATTR_INDX_MEMORY_WRITE_THROUGH: - GcdAttributes = EFI_MEMORY_WT; - break; - case TT_ATTR_INDX_MEMORY_WRITE_BACK: - GcdAttributes = EFI_MEMORY_WB; - break; - default: - DEBUG ((EFI_D_ERROR, "PageAttributeToGcdAttribute: PageAttributes:0x%lX not supported.\n", PageAttributes)); - ASSERT (0); - // The Global Coherency Domain (GCD) value is defined as a bit set. - // Returning 0 means no attribute has been set. - GcdAttributes = 0; - } - - // Determine protection attributes - if (((PageAttributes & TT_AP_MASK) == TT_AP_NO_RO) || ((PageAttributes & TT_AP_MASK) == TT_AP_RO_RO)) { - // Read only cases map to write-protect - GcdAttributes |= EFI_MEMORY_WP; - } - - // Process eXecute Never attribute - if ((PageAttributes & (TT_PXN_MASK | TT_UXN_MASK)) != 0 ) { - GcdAttributes |= EFI_MEMORY_XP; - } - - return GcdAttributes; -} - -ARM_MEMORY_REGION_ATTRIBUTES -GcdAttributeToArmAttribute ( - IN UINT64 GcdAttributes - ) -{ - switch (GcdAttributes & 0xFF) { - case EFI_MEMORY_UC: - return ARM_MEMORY_REGION_ATTRIBUTE_DEVICE; - case EFI_MEMORY_WC: - return ARM_MEMORY_REGION_ATTRIBUTE_UNCACHED_UNBUFFERED; - case EFI_MEMORY_WT: - return ARM_MEMORY_REGION_ATTRIBUTE_WRITE_THROUGH; - case EFI_MEMORY_WB: - return ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK; - default: - DEBUG ((EFI_D_ERROR, "GcdAttributeToArmAttribute: 0x%lX attributes is not supported.\n", GcdAttributes)); - ASSERT (0); - return ARM_MEMORY_REGION_ATTRIBUTE_DEVICE; - } -} - -// Describe the T0SZ values for each translation table level -typedef struct { - UINTN MinT0SZ; - UINTN MaxT0SZ; - UINTN LargestT0SZ; // Generally (MaxT0SZ == LargestT0SZ) but at the Level3 Table - // the MaxT0SZ is not at the boundary of the table -} T0SZ_DESCRIPTION_PER_LEVEL; - -// Map table for the corresponding Level of Table -STATIC CONST T0SZ_DESCRIPTION_PER_LEVEL T0SZPerTableLevel[] = { - { 16, 24, 24 }, // Table Level 0 - { 25, 33, 33 }, // Table Level 1 - { 34, 39, 42 } // Table Level 2 -}; +#define MIN_T0SZ 16 +#define BITS_PER_LEVEL 9 VOID GetRootTranslationTableInfo ( @@ -143,307 +67,285 @@ GetRootTranslationTableInfo ( OUT UINTN *TableEntryCount ) { - UINTN Index; - - // Identify the level of the root table from the given T0SZ - for (Index = 0; Index < sizeof (T0SZPerTableLevel) / sizeof (T0SZ_DESCRIPTION_PER_LEVEL); Index++) { - if (T0SZ <= T0SZPerTableLevel[Index].MaxT0SZ) { - break; - } - } - - // If we have not found the corresponding maximum T0SZ then we use the last one - if (Index == sizeof (T0SZPerTableLevel) / sizeof (T0SZ_DESCRIPTION_PER_LEVEL)) { - Index--; - } - // Get the level of the root table if (TableLevel) { - *TableLevel = Index; + *TableLevel = (T0SZ - MIN_T0SZ) / BITS_PER_LEVEL; } - // The Size of the Table is 2^(T0SZ-LargestT0SZ) if (TableEntryCount) { - *TableEntryCount = 1 << (T0SZPerTableLevel[Index].LargestT0SZ - T0SZ + 1); + *TableEntryCount = 1UL << (BITS_PER_LEVEL - (T0SZ - MIN_T0SZ) % BITS_PER_LEVEL); } } STATIC VOID -ReplaceLiveEntry ( +ReplaceTableEntry ( IN UINT64 *Entry, - IN UINT64 Value + IN UINT64 Value, + IN UINT64 RegionStart, + IN BOOLEAN IsLiveBlockMapping ) { - if (!ArmMmuEnabled ()) { + if (!ArmMmuEnabled () || !IsLiveBlockMapping) { *Entry = Value; + ArmUpdateTranslationTableEntry (Entry, (VOID *)(UINTN)RegionStart); } else { - ArmReplaceLiveTranslationEntry (Entry, Value); + ArmReplaceLiveTranslationEntry (Entry, Value, RegionStart); } } STATIC VOID -LookupAddresstoRootTable ( - IN UINT64 MaxAddress, - OUT UINTN *T0SZ, - OUT UINTN *TableEntryCount +FreePageTablesRecursive ( + IN UINT64 *TranslationTable, + IN UINTN Level ) { - UINTN TopBit; + UINTN Index; - // Check the parameters are not NULL - ASSERT ((T0SZ != NULL) && (TableEntryCount != NULL)); + ASSERT (Level <= 3); - // Look for the highest bit set in MaxAddress - for (TopBit = 63; TopBit != 0; TopBit--) { - if ((1ULL << TopBit) & MaxAddress) { - // MaxAddress top bit is found - TopBit = TopBit + 1; - break; + if (Level < 3) { + for (Index = 0; Index < TT_ENTRY_COUNT; Index++) { + if ((TranslationTable[Index] & TT_TYPE_MASK) == TT_TYPE_TABLE_ENTRY) { + FreePageTablesRecursive ((VOID *)(UINTN)(TranslationTable[Index] & + TT_ADDRESS_MASK_BLOCK_ENTRY), + Level + 1); + } } } - ASSERT (TopBit != 0); - - // Calculate T0SZ from the top bit of the MaxAddress - *T0SZ = 64 - TopBit; - - // Get the Table info from T0SZ - GetRootTranslationTableInfo (*T0SZ, NULL, TableEntryCount); + FreePages (TranslationTable, 1); } STATIC -UINT64* -GetBlockEntryListFromAddress ( - IN UINT64 *RootTable, - IN UINT64 RegionStart, - OUT UINTN *TableLevel, - IN OUT UINT64 *BlockEntrySize, - OUT UINT64 **LastBlockEntry +BOOLEAN +IsBlockEntry ( + IN UINT64 Entry, + IN UINTN Level ) { - UINTN RootTableLevel; - UINTN RootTableEntryCount; - UINT64 *TranslationTable; - UINT64 *BlockEntry; - UINT64 *SubTableBlockEntry; - UINT64 BlockEntryAddress; - UINTN BaseAddressAlignment; - UINTN PageLevel; - UINTN Index; - UINTN IndexLevel; - UINTN T0SZ; - UINT64 Attributes; - UINT64 TableAttributes; - - // Initialize variable - BlockEntry = NULL; - - // Ensure the parameters are valid - if (!(TableLevel && BlockEntrySize && LastBlockEntry)) { - ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER); - return NULL; + if (Level == 3) { + return (Entry & TT_TYPE_MASK) == TT_TYPE_BLOCK_ENTRY_LEVEL3; } + return (Entry & TT_TYPE_MASK) == TT_TYPE_BLOCK_ENTRY; +} - // Ensure the Region is aligned on 4KB boundary - if ((RegionStart & (SIZE_4KB - 1)) != 0) { - ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER); - return NULL; - } - - // Ensure the required size is aligned on 4KB boundary and not 0 - if ((*BlockEntrySize & (SIZE_4KB - 1)) != 0 || *BlockEntrySize == 0) { - ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER); - return NULL; - } - - T0SZ = ArmGetTCR () & TCR_T0SZ_MASK; - // Get the Table info from T0SZ - GetRootTranslationTableInfo (T0SZ, &RootTableLevel, &RootTableEntryCount); - - // If the start address is 0x0 then we use the size of the region to identify the alignment - if (RegionStart == 0) { - // Identify the highest possible alignment for the Region Size - BaseAddressAlignment = LowBitSet64 (*BlockEntrySize); - } else { - // Identify the highest possible alignment for the Base Address - BaseAddressAlignment = LowBitSet64 (RegionStart); - } - - // Identify the Page Level the RegionStart must belong to. Note that PageLevel - // should be at least 1 since block translations are not supported at level 0 - PageLevel = MAX (3 - ((BaseAddressAlignment - 12) / 9), 1); - - // If the required size is smaller than the current block size then we need to go to the page below. - // The PageLevel was calculated on the Base Address alignment but did not take in account the alignment - // of the allocation size - while (*BlockEntrySize < TT_BLOCK_ENTRY_SIZE_AT_LEVEL (PageLevel)) { - // It does not fit so we need to go a page level above - PageLevel++; +STATIC +BOOLEAN +IsTableEntry ( + IN UINT64 Entry, + IN UINTN Level + ) +{ + if (Level == 3) { + // + // TT_TYPE_TABLE_ENTRY aliases TT_TYPE_BLOCK_ENTRY_LEVEL3 + // so we need to take the level into account as well. + // + return FALSE; } + return (Entry & TT_TYPE_MASK) == TT_TYPE_TABLE_ENTRY; +} - // - // Get the Table Descriptor for the corresponding PageLevel. We need to decompose RegionStart to get appropriate entries - // - - TranslationTable = RootTable; - for (IndexLevel = RootTableLevel; IndexLevel <= PageLevel; IndexLevel++) { - BlockEntry = (UINT64*)TT_GET_ENTRY_FOR_ADDRESS (TranslationTable, IndexLevel, RegionStart); - - if ((IndexLevel != 3) && ((*BlockEntry & TT_TYPE_MASK) == TT_TYPE_TABLE_ENTRY)) { - // Go to the next table - TranslationTable = (UINT64*)(*BlockEntry & TT_ADDRESS_MASK_DESCRIPTION_TABLE); - - // If we are at the last level then update the last level to next level - if (IndexLevel == PageLevel) { - // Enter the next level - PageLevel++; - } - } else if ((*BlockEntry & TT_TYPE_MASK) == TT_TYPE_BLOCK_ENTRY) { - // If we are not at the last level then we need to split this BlockEntry - if (IndexLevel != PageLevel) { - // Retrieve the attributes from the block entry - Attributes = *BlockEntry & TT_ATTRIBUTES_MASK; - - // Convert the block entry attributes into Table descriptor attributes - TableAttributes = TT_TABLE_AP_NO_PERMISSION; - if (Attributes & TT_NS) { - TableAttributes = TT_TABLE_NS; +STATIC +EFI_STATUS +UpdateRegionMappingRecursive ( + IN UINT64 RegionStart, + IN UINT64 RegionEnd, + IN UINT64 AttributeSetMask, + IN UINT64 AttributeClearMask, + IN UINT64 *PageTable, + IN UINTN Level + ) +{ + UINTN BlockShift; + UINT64 BlockMask; + UINT64 BlockEnd; + UINT64 *Entry; + UINT64 EntryValue; + VOID *TranslationTable; + EFI_STATUS Status; + + ASSERT (((RegionStart | RegionEnd) & EFI_PAGE_MASK) == 0); + + BlockShift = (Level + 1) * BITS_PER_LEVEL + MIN_T0SZ; + BlockMask = MAX_UINT64 >> BlockShift; + + DEBUG ((DEBUG_VERBOSE, "%a(%d): %llx - %llx set %lx clr %lx\n", __FUNCTION__, + Level, RegionStart, RegionEnd, AttributeSetMask, AttributeClearMask)); + + for (; RegionStart < RegionEnd; RegionStart = BlockEnd) { + BlockEnd = MIN (RegionEnd, (RegionStart | BlockMask) + 1); + Entry = &PageTable[(RegionStart >> (64 - BlockShift)) & (TT_ENTRY_COUNT - 1)]; + + // + // If RegionStart or BlockEnd is not aligned to the block size at this + // level, we will have to create a table mapping in order to map less + // than a block, and recurse to create the block or page entries at + // the next level. No block mappings are allowed at all at level 0, + // so in that case, we have to recurse unconditionally. + // If we are changing a table entry and the AttributeClearMask is non-zero, + // we cannot replace it with a block entry without potentially losing + // attribute information, so keep the table entry in that case. + // + if (Level == 0 || ((RegionStart | BlockEnd) & BlockMask) != 0 || + (IsTableEntry (*Entry, Level) && AttributeClearMask != 0)) { + ASSERT (Level < 3); + + if (!IsTableEntry (*Entry, Level)) { + // + // No table entry exists yet, so we need to allocate a page table + // for the next level. + // + TranslationTable = AllocatePages (1); + if (TranslationTable == NULL) { + return EFI_OUT_OF_RESOURCES; } - // Get the address corresponding at this entry - BlockEntryAddress = RegionStart; - BlockEntryAddress = BlockEntryAddress >> TT_ADDRESS_OFFSET_AT_LEVEL(IndexLevel); - // Shift back to right to set zero before the effective address - BlockEntryAddress = BlockEntryAddress << TT_ADDRESS_OFFSET_AT_LEVEL(IndexLevel); - - // Set the correct entry type for the next page level - if ((IndexLevel + 1) == 3) { - Attributes |= TT_TYPE_BLOCK_ENTRY_LEVEL3; - } else { - Attributes |= TT_TYPE_BLOCK_ENTRY; + if (!ArmMmuEnabled ()) { + // + // Make sure we are not inadvertently hitting in the caches + // when populating the page tables. + // + InvalidateDataCacheRange (TranslationTable, EFI_PAGE_SIZE); } - // Create a new translation table - TranslationTable = (UINT64*)AllocateAlignedPages (EFI_SIZE_TO_PAGES(TT_ENTRY_COUNT * sizeof(UINT64)), TT_ALIGNMENT_DESCRIPTION_TABLE); - if (TranslationTable == NULL) { - return NULL; + ZeroMem (TranslationTable, EFI_PAGE_SIZE); + + if (IsBlockEntry (*Entry, Level)) { + // + // We are splitting an existing block entry, so we have to populate + // the new table with the attributes of the block entry it replaces. + // + Status = UpdateRegionMappingRecursive (RegionStart & ~BlockMask, + (RegionStart | BlockMask) + 1, *Entry & TT_ATTRIBUTES_MASK, + 0, TranslationTable, Level + 1); + if (EFI_ERROR (Status)) { + // + // The range we passed to UpdateRegionMappingRecursive () is block + // aligned, so it is guaranteed that no further pages were allocated + // by it, and so we only have to free the page we allocated here. + // + FreePages (TranslationTable, 1); + return Status; + } } + } else { + TranslationTable = (VOID *)(UINTN)(*Entry & TT_ADDRESS_MASK_BLOCK_ENTRY); + } - // Populate the newly created lower level table - SubTableBlockEntry = TranslationTable; - for (Index = 0; Index < TT_ENTRY_COUNT; Index++) { - *SubTableBlockEntry = Attributes | (BlockEntryAddress + (Index << TT_ADDRESS_OFFSET_AT_LEVEL(IndexLevel + 1))); - SubTableBlockEntry++; + // + // Recurse to the next level + // + Status = UpdateRegionMappingRecursive (RegionStart, BlockEnd, + AttributeSetMask, AttributeClearMask, TranslationTable, + Level + 1); + if (EFI_ERROR (Status)) { + if (!IsTableEntry (*Entry, Level)) { + // + // We are creating a new table entry, so on failure, we can free all + // allocations we made recursively, given that the whole subhierarchy + // has not been wired into the live page tables yet. (This is not + // possible for existing table entries, since we cannot revert the + // modifications we made to the subhierarchy it represents.) + // + FreePageTablesRecursive (TranslationTable, Level + 1); } + return Status; + } - // Fill the BlockEntry with the new TranslationTable - ReplaceLiveEntry (BlockEntry, - ((UINTN)TranslationTable & TT_ADDRESS_MASK_DESCRIPTION_TABLE) | TableAttributes | TT_TYPE_TABLE_ENTRY); + if (!IsTableEntry (*Entry, Level)) { + EntryValue = (UINTN)TranslationTable | TT_TYPE_TABLE_ENTRY; + ReplaceTableEntry (Entry, EntryValue, RegionStart, + IsBlockEntry (*Entry, Level)); } } else { - if (IndexLevel != PageLevel) { + EntryValue = (*Entry & AttributeClearMask) | AttributeSetMask; + EntryValue |= RegionStart; + EntryValue |= (Level == 3) ? TT_TYPE_BLOCK_ENTRY_LEVEL3 + : TT_TYPE_BLOCK_ENTRY; + + if (IsTableEntry (*Entry, Level)) { // - // Case when we have an Invalid Entry and we are at a page level above of the one targetted. + // We are replacing a table entry with a block entry. This is only + // possible if we are keeping none of the original attributes. + // We can free the table entry's page table, and all the ones below + // it, since we are dropping the only possible reference to it. // - - // Create a new translation table - TranslationTable = (UINT64*)AllocateAlignedPages (EFI_SIZE_TO_PAGES(TT_ENTRY_COUNT * sizeof(UINT64)), TT_ALIGNMENT_DESCRIPTION_TABLE); - if (TranslationTable == NULL) { - return NULL; - } - - ZeroMem (TranslationTable, TT_ENTRY_COUNT * sizeof(UINT64)); - - // Fill the new BlockEntry with the TranslationTable - *BlockEntry = ((UINTN)TranslationTable & TT_ADDRESS_MASK_DESCRIPTION_TABLE) | TT_TYPE_TABLE_ENTRY; + ASSERT (AttributeClearMask == 0); + TranslationTable = (VOID *)(UINTN)(*Entry & TT_ADDRESS_MASK_BLOCK_ENTRY); + ReplaceTableEntry (Entry, EntryValue, RegionStart, TRUE); + FreePageTablesRecursive (TranslationTable, Level + 1); + } else { + ReplaceTableEntry (Entry, EntryValue, RegionStart, FALSE); } } } + return EFI_SUCCESS; +} - // Expose the found PageLevel to the caller - *TableLevel = PageLevel; +STATIC +VOID +LookupAddresstoRootTable ( + IN UINT64 MaxAddress, + OUT UINTN *T0SZ, + OUT UINTN *TableEntryCount + ) +{ + UINTN TopBit; + + // Check the parameters are not NULL + ASSERT ((T0SZ != NULL) && (TableEntryCount != NULL)); - // Now, we have the Table Level we can get the Block Size associated to this table - *BlockEntrySize = TT_BLOCK_ENTRY_SIZE_AT_LEVEL (PageLevel); + // Look for the highest bit set in MaxAddress + for (TopBit = 63; TopBit != 0; TopBit--) { + if ((1ULL << TopBit) & MaxAddress) { + // MaxAddress top bit is found + TopBit = TopBit + 1; + break; + } + } + ASSERT (TopBit != 0); - // The last block of the root table depends on the number of entry in this table, - // otherwise it is always the (TT_ENTRY_COUNT - 1)th entry in the table. - *LastBlockEntry = TT_LAST_BLOCK_ADDRESS(TranslationTable, - (PageLevel == RootTableLevel) ? RootTableEntryCount : TT_ENTRY_COUNT); + // Calculate T0SZ from the top bit of the MaxAddress + *T0SZ = 64 - TopBit; - return BlockEntry; + // Get the Table info from T0SZ + GetRootTranslationTableInfo (*T0SZ, NULL, TableEntryCount); } STATIC -RETURN_STATUS +EFI_STATUS UpdateRegionMapping ( - IN UINT64 *RootTable, IN UINT64 RegionStart, IN UINT64 RegionLength, - IN UINT64 Attributes, - IN UINT64 BlockEntryMask + IN UINT64 AttributeSetMask, + IN UINT64 AttributeClearMask ) { - UINT32 Type; - UINT64 *BlockEntry; - UINT64 *LastBlockEntry; - UINT64 BlockEntrySize; - UINTN TableLevel; - - // Ensure the Length is aligned on 4KB boundary - if ((RegionLength == 0) || ((RegionLength & (SIZE_4KB - 1)) != 0)) { - ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER); - return RETURN_INVALID_PARAMETER; - } - - do { - // Get the first Block Entry that matches the Virtual Address and also the information on the Table Descriptor - // such as the the size of the Block Entry and the address of the last BlockEntry of the Table Descriptor - BlockEntrySize = RegionLength; - BlockEntry = GetBlockEntryListFromAddress (RootTable, RegionStart, &TableLevel, &BlockEntrySize, &LastBlockEntry); - if (BlockEntry == NULL) { - // GetBlockEntryListFromAddress() return NULL when it fails to allocate new pages from the Translation Tables - return RETURN_OUT_OF_RESOURCES; - } + UINTN RootTableLevel; + UINTN T0SZ; - if (TableLevel != 3) { - Type = TT_TYPE_BLOCK_ENTRY; - } else { - Type = TT_TYPE_BLOCK_ENTRY_LEVEL3; - } + if (((RegionStart | RegionLength) & EFI_PAGE_MASK)) { + return EFI_INVALID_PARAMETER; + } - do { - // Fill the Block Entry with attribute and output block address - *BlockEntry &= BlockEntryMask; - *BlockEntry |= (RegionStart & TT_ADDRESS_MASK_BLOCK_ENTRY) | Attributes | Type; - - // Go to the next BlockEntry - RegionStart += BlockEntrySize; - RegionLength -= BlockEntrySize; - BlockEntry++; - - // Break the inner loop when next block is a table - // Rerun GetBlockEntryListFromAddress to avoid page table memory leak - if (TableLevel != 3 && - (*BlockEntry & TT_TYPE_MASK) == TT_TYPE_TABLE_ENTRY) { - break; - } - } while ((RegionLength >= BlockEntrySize) && (BlockEntry <= LastBlockEntry)); - } while (RegionLength != 0); + T0SZ = ArmGetTCR () & TCR_T0SZ_MASK; + GetRootTranslationTableInfo (T0SZ, &RootTableLevel, NULL); - return RETURN_SUCCESS; + return UpdateRegionMappingRecursive (RegionStart, RegionStart + RegionLength, + AttributeSetMask, AttributeClearMask, ArmGetTTBR0BaseAddress (), + RootTableLevel); } STATIC -RETURN_STATUS +EFI_STATUS FillTranslationTable ( IN UINT64 *RootTable, IN ARM_MEMORY_REGION_DESCRIPTOR *MemoryRegion ) { return UpdateRegionMapping ( - RootTable, MemoryRegion->VirtualBase, MemoryRegion->Length, ArmMemoryAttributeToPageAttribute (MemoryRegion->Attributes) | TT_AF, @@ -451,38 +353,77 @@ FillTranslationTable ( ); } -RETURN_STATUS -SetMemoryAttributes ( - IN EFI_PHYSICAL_ADDRESS BaseAddress, - IN UINT64 Length, - IN UINT64 Attributes, - IN EFI_PHYSICAL_ADDRESS VirtualMask +STATIC +UINT64 +GcdAttributeToPageAttribute ( + IN UINT64 GcdAttributes ) { - RETURN_STATUS Status; - ARM_MEMORY_REGION_DESCRIPTOR MemoryRegion; - UINT64 *TranslationTable; + UINT64 PageAttributes; - MemoryRegion.PhysicalBase = BaseAddress; - MemoryRegion.VirtualBase = BaseAddress; - MemoryRegion.Length = Length; - MemoryRegion.Attributes = GcdAttributeToArmAttribute (Attributes); + switch (GcdAttributes & EFI_MEMORY_CACHETYPE_MASK) { + case EFI_MEMORY_UC: + PageAttributes = TT_ATTR_INDX_DEVICE_MEMORY; + break; + case EFI_MEMORY_WC: + PageAttributes = TT_ATTR_INDX_MEMORY_NON_CACHEABLE; + break; + case EFI_MEMORY_WT: + PageAttributes = TT_ATTR_INDX_MEMORY_WRITE_THROUGH | TT_SH_INNER_SHAREABLE; + break; + case EFI_MEMORY_WB: + PageAttributes = TT_ATTR_INDX_MEMORY_WRITE_BACK | TT_SH_INNER_SHAREABLE; + break; + default: + PageAttributes = TT_ATTR_INDX_MASK; + break; + } - TranslationTable = ArmGetTTBR0BaseAddress (); + if ((GcdAttributes & EFI_MEMORY_XP) != 0 || + (GcdAttributes & EFI_MEMORY_CACHETYPE_MASK) == EFI_MEMORY_UC) { + if (ArmReadCurrentEL () == AARCH64_EL2) { + PageAttributes |= TT_XN_MASK; + } else { + PageAttributes |= TT_UXN_MASK | TT_PXN_MASK; + } + } - Status = FillTranslationTable (TranslationTable, &MemoryRegion); - if (RETURN_ERROR (Status)) { - return Status; + if ((GcdAttributes & EFI_MEMORY_RO) != 0) { + PageAttributes |= TT_AP_RO_RO; } - // Invalidate all TLB entries so changes are synced - ArmInvalidateTlb (); + return PageAttributes | TT_AF; +} - return RETURN_SUCCESS; +EFI_STATUS +ArmSetMemoryAttributes ( + IN EFI_PHYSICAL_ADDRESS BaseAddress, + IN UINT64 Length, + IN UINT64 Attributes + ) +{ + UINT64 PageAttributes; + UINT64 PageAttributeMask; + + PageAttributes = GcdAttributeToPageAttribute (Attributes); + PageAttributeMask = 0; + + if ((Attributes & EFI_MEMORY_CACHETYPE_MASK) == 0) { + // + // No memory type was set in Attributes, so we are going to update the + // permissions only. + // + PageAttributes &= TT_AP_MASK | TT_UXN_MASK | TT_PXN_MASK; + PageAttributeMask = ~(TT_ADDRESS_MASK_BLOCK_ENTRY | TT_AP_MASK | + TT_PXN_MASK | TT_XN_MASK); + } + + return UpdateRegionMapping (BaseAddress, Length, PageAttributes, + PageAttributeMask); } STATIC -RETURN_STATUS +EFI_STATUS SetMemoryRegionAttribute ( IN EFI_PHYSICAL_ADDRESS BaseAddress, IN UINT64 Length, @@ -490,23 +431,10 @@ SetMemoryRegionAttribute ( IN UINT64 BlockEntryMask ) { - RETURN_STATUS Status; - UINT64 *RootTable; - - RootTable = ArmGetTTBR0BaseAddress (); - - Status = UpdateRegionMapping (RootTable, BaseAddress, Length, Attributes, BlockEntryMask); - if (RETURN_ERROR (Status)) { - return Status; - } - - // Invalidate all TLB entries so changes are synced - ArmInvalidateTlb (); - - return RETURN_SUCCESS; + return UpdateRegionMapping (BaseAddress, Length, Attributes, BlockEntryMask); } -RETURN_STATUS +EFI_STATUS ArmSetMemoryRegionNoExec ( IN EFI_PHYSICAL_ADDRESS BaseAddress, IN UINT64 Length @@ -527,7 +455,7 @@ ArmSetMemoryRegionNoExec ( ~TT_ADDRESS_MASK_BLOCK_ENTRY); } -RETURN_STATUS +EFI_STATUS ArmClearMemoryRegionNoExec ( IN EFI_PHYSICAL_ADDRESS BaseAddress, IN UINT64 Length @@ -545,7 +473,7 @@ ArmClearMemoryRegionNoExec ( Mask); } -RETURN_STATUS +EFI_STATUS ArmSetMemoryRegionReadOnly ( IN EFI_PHYSICAL_ADDRESS BaseAddress, IN UINT64 Length @@ -558,7 +486,7 @@ ArmSetMemoryRegionReadOnly ( ~TT_ADDRESS_MASK_BLOCK_ENTRY); } -RETURN_STATUS +EFI_STATUS ArmClearMemoryRegionReadOnly ( IN EFI_PHYSICAL_ADDRESS BaseAddress, IN UINT64 Length @@ -571,7 +499,7 @@ ArmClearMemoryRegionReadOnly ( ~(TT_ADDRESS_MASK_BLOCK_ENTRY | TT_AP_MASK)); } -RETURN_STATUS +EFI_STATUS EFIAPI ArmConfigureMmu ( IN ARM_MEMORY_REGION_DESCRIPTOR *MemoryTable, @@ -580,31 +508,26 @@ ArmConfigureMmu ( ) { VOID* TranslationTable; - UINTN TranslationTablePageCount; - UINT32 TranslationTableAttribute; - ARM_MEMORY_REGION_DESCRIPTOR *MemoryTableEntry; UINT64 MaxAddress; - UINT64 TopAddress; UINTN T0SZ; UINTN RootTableEntryCount; UINT64 TCR; - RETURN_STATUS Status; + EFI_STATUS Status; - if(MemoryTable == NULL) { + if (MemoryTable == NULL) { ASSERT (MemoryTable != NULL); - return RETURN_INVALID_PARAMETER; + return EFI_INVALID_PARAMETER; } - // Identify the highest address of the memory table - MaxAddress = MemoryTable->PhysicalBase + MemoryTable->Length - 1; - MemoryTableEntry = MemoryTable; - while (MemoryTableEntry->Length != 0) { - TopAddress = MemoryTableEntry->PhysicalBase + MemoryTableEntry->Length - 1; - if (TopAddress > MaxAddress) { - MaxAddress = TopAddress; - } - MemoryTableEntry++; - } + // + // Limit the virtual address space to what we can actually use: UEFI + // mandates a 1:1 mapping, so no point in making the virtual address + // space larger than the physical address space. We also have to take + // into account the architectural limitations that result from UEFI's + // use of 4 KB pages. + // + MaxAddress = MIN (LShiftU64 (1ULL, ArmGetPhysicalAddressBits ()) - 1, + MAX_ALLOC_ADDRESS); // Lookup the Table Level to get the information LookupAddresstoRootTable (MaxAddress, &T0SZ, &RootTableEntryCount); @@ -632,9 +555,11 @@ ArmConfigureMmu ( } else if (MaxAddress < SIZE_256TB) { TCR |= TCR_PS_256TB; } else { - DEBUG ((EFI_D_ERROR, "ArmConfigureMmu: The MaxAddress 0x%lX is not supported by this MMU configuration.\n", MaxAddress)); + DEBUG ((DEBUG_ERROR, + "ArmConfigureMmu: The MaxAddress 0x%lX is not supported by this MMU configuration.\n", + MaxAddress)); ASSERT (0); // Bigger than 48-bit memory space are not supported - return RETURN_UNSUPPORTED; + return EFI_UNSUPPORTED; } } else if (ArmReadCurrentEL () == AARCH64_EL1) { // Due to Cortex-A57 erratum #822227 we must set TG1[1] == 1, regardless of EPD1. @@ -654,27 +579,44 @@ ArmConfigureMmu ( } else if (MaxAddress < SIZE_256TB) { TCR |= TCR_IPS_256TB; } else { - DEBUG ((EFI_D_ERROR, "ArmConfigureMmu: The MaxAddress 0x%lX is not supported by this MMU configuration.\n", MaxAddress)); + DEBUG ((DEBUG_ERROR, + "ArmConfigureMmu: The MaxAddress 0x%lX is not supported by this MMU configuration.\n", + MaxAddress)); ASSERT (0); // Bigger than 48-bit memory space are not supported - return RETURN_UNSUPPORTED; + return EFI_UNSUPPORTED; } } else { ASSERT (0); // UEFI is only expected to run at EL2 and EL1, not EL3. - return RETURN_UNSUPPORTED; + return EFI_UNSUPPORTED; } + // + // Translation table walks are always cache coherent on ARMv8-A, so cache + // maintenance on page tables is never needed. Since there is a risk of + // loss of coherency when using mismatched attributes, and given that memory + // is mapped cacheable except for extraordinary cases (such as non-coherent + // DMA), have the page table walker perform cached accesses as well, and + // assert below that that matches the attributes we use for CPU accesses to + // the region. + // + TCR |= TCR_SH_INNER_SHAREABLE | + TCR_RGN_OUTER_WRITE_BACK_ALLOC | + TCR_RGN_INNER_WRITE_BACK_ALLOC; + // Set TCR ArmSetTCR (TCR); // Allocate pages for translation table - TranslationTablePageCount = EFI_SIZE_TO_PAGES(RootTableEntryCount * sizeof(UINT64)); - TranslationTable = (UINT64*)AllocateAlignedPages (TranslationTablePageCount, TT_ALIGNMENT_DESCRIPTION_TABLE); + TranslationTable = AllocatePages (1); if (TranslationTable == NULL) { - return RETURN_OUT_OF_RESOURCES; + return EFI_OUT_OF_RESOURCES; } - // We set TTBR0 just after allocating the table to retrieve its location from the subsequent - // functions without needing to pass this value across the functions. The MMU is only enabled - // after the translation tables are populated. + // + // We set TTBR0 just after allocating the table to retrieve its location from + // the subsequent functions without needing to pass this value across the + // functions. The MMU is only enabled after the translation tables are + // populated. + // ArmSetTTBR0 (TranslationTable); if (TranslationTableBase != NULL) { @@ -682,70 +624,48 @@ ArmConfigureMmu ( } if (TranslationTableSize != NULL) { - *TranslationTableSize = RootTableEntryCount * sizeof(UINT64); + *TranslationTableSize = RootTableEntryCount * sizeof (UINT64); } - ZeroMem (TranslationTable, RootTableEntryCount * sizeof(UINT64)); - - // Disable MMU and caches. ArmDisableMmu() also invalidates the TLBs - ArmDisableMmu (); - ArmDisableDataCache (); - ArmDisableInstructionCache (); - - // Make sure nothing sneaked into the cache - ArmCleanInvalidateDataCache (); - ArmInvalidateInstructionCache (); + // + // Make sure we are not inadvertently hitting in the caches + // when populating the page tables. + // + InvalidateDataCacheRange (TranslationTable, + RootTableEntryCount * sizeof (UINT64)); + ZeroMem (TranslationTable, RootTableEntryCount * sizeof (UINT64)); - TranslationTableAttribute = TT_ATTR_INDX_INVALID; while (MemoryTable->Length != 0) { - // Find the memory attribute for the Translation Table - if (((UINTN)TranslationTable >= MemoryTable->PhysicalBase) && - ((UINTN)TranslationTable <= MemoryTable->PhysicalBase - 1 + MemoryTable->Length)) { - TranslationTableAttribute = MemoryTable->Attributes; - } - Status = FillTranslationTable (TranslationTable, MemoryTable); - if (RETURN_ERROR (Status)) { - goto FREE_TRANSLATION_TABLE; + if (EFI_ERROR (Status)) { + goto FreeTranslationTable; } MemoryTable++; } - // Translate the Memory Attributes into Translation Table Register Attributes - if ((TranslationTableAttribute == ARM_MEMORY_REGION_ATTRIBUTE_UNCACHED_UNBUFFERED) || - (TranslationTableAttribute == ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_UNCACHED_UNBUFFERED)) { - TCR |= TCR_SH_NON_SHAREABLE | TCR_RGN_OUTER_NON_CACHEABLE | TCR_RGN_INNER_NON_CACHEABLE; - } else if ((TranslationTableAttribute == ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK) || - (TranslationTableAttribute == ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK)) { - TCR |= TCR_SH_INNER_SHAREABLE | TCR_RGN_OUTER_WRITE_BACK_ALLOC | TCR_RGN_INNER_WRITE_BACK_ALLOC; - } else if ((TranslationTableAttribute == ARM_MEMORY_REGION_ATTRIBUTE_WRITE_THROUGH) || - (TranslationTableAttribute == ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_THROUGH)) { - TCR |= TCR_SH_NON_SHAREABLE | TCR_RGN_OUTER_WRITE_THROUGH | TCR_RGN_INNER_WRITE_THROUGH; - } else { - // If we failed to find a mapping that contains the root translation table then it probably means the translation table - // is not mapped in the given memory map. - ASSERT (0); - Status = RETURN_UNSUPPORTED; - goto FREE_TRANSLATION_TABLE; - } - - // Set again TCR after getting the Translation Table attributes - ArmSetTCR (TCR); - - ArmSetMAIR (MAIR_ATTR(TT_ATTR_INDX_DEVICE_MEMORY, MAIR_ATTR_DEVICE_MEMORY) | // mapped to EFI_MEMORY_UC - MAIR_ATTR(TT_ATTR_INDX_MEMORY_NON_CACHEABLE, MAIR_ATTR_NORMAL_MEMORY_NON_CACHEABLE) | // mapped to EFI_MEMORY_WC - MAIR_ATTR(TT_ATTR_INDX_MEMORY_WRITE_THROUGH, MAIR_ATTR_NORMAL_MEMORY_WRITE_THROUGH) | // mapped to EFI_MEMORY_WT - MAIR_ATTR(TT_ATTR_INDX_MEMORY_WRITE_BACK, MAIR_ATTR_NORMAL_MEMORY_WRITE_BACK)); // mapped to EFI_MEMORY_WB + // + // EFI_MEMORY_UC ==> MAIR_ATTR_DEVICE_MEMORY + // EFI_MEMORY_WC ==> MAIR_ATTR_NORMAL_MEMORY_NON_CACHEABLE + // EFI_MEMORY_WT ==> MAIR_ATTR_NORMAL_MEMORY_WRITE_THROUGH + // EFI_MEMORY_WB ==> MAIR_ATTR_NORMAL_MEMORY_WRITE_BACK + // + ArmSetMAIR ( + MAIR_ATTR (TT_ATTR_INDX_DEVICE_MEMORY, MAIR_ATTR_DEVICE_MEMORY) | + MAIR_ATTR (TT_ATTR_INDX_MEMORY_NON_CACHEABLE, MAIR_ATTR_NORMAL_MEMORY_NON_CACHEABLE) | + MAIR_ATTR (TT_ATTR_INDX_MEMORY_WRITE_THROUGH, MAIR_ATTR_NORMAL_MEMORY_WRITE_THROUGH) | + MAIR_ATTR (TT_ATTR_INDX_MEMORY_WRITE_BACK, MAIR_ATTR_NORMAL_MEMORY_WRITE_BACK) + ); ArmDisableAlignmentCheck (); + ArmEnableStackAlignmentCheck (); ArmEnableInstructionCache (); ArmEnableDataCache (); ArmEnableMmu (); - return RETURN_SUCCESS; + return EFI_SUCCESS; -FREE_TRANSLATION_TABLE: - FreePages (TranslationTable, TranslationTablePageCount); +FreeTranslationTable: + FreePages (TranslationTable, 1); return Status; }