/** @file\r
* File managing the MMU for ARMv8 architecture\r
*\r
-* Copyright (c) 2011-2014, ARM Limited. All rights reserved.\r
+* Copyright (c) 2011-2020, ARM Limited. All rights reserved.\r
* Copyright (c) 2016, Linaro Limited. All rights reserved.\r
+* Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>\r
*\r
-* This program and the accompanying materials\r
-* are licensed and made available under the terms and conditions of the BSD License\r
-* which accompanies this distribution. The full text of the license may be found at\r
-* http://opensource.org/licenses/bsd-license.php\r
-*\r
-* THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
-* WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
+* SPDX-License-Identifier: BSD-2-Clause-Patent\r
*\r
**/\r
\r
)\r
{\r
switch (Attributes) {\r
+ case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK_NONSHAREABLE:\r
+ case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK_NONSHAREABLE:\r
+ return TT_ATTR_INDX_MEMORY_WRITE_BACK;\r
+\r
case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK:\r
case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK:\r
return TT_ATTR_INDX_MEMORY_WRITE_BACK | TT_SH_INNER_SHAREABLE;\r
return TT_ATTR_INDX_MEMORY_NON_CACHEABLE;\r
\r
default:\r
- ASSERT(0);\r
+ ASSERT (0);\r
case ARM_MEMORY_REGION_ATTRIBUTE_DEVICE:\r
case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_DEVICE:\r
if (ArmReadCurrentEL () == AARCH64_EL2)\r
}\r
}\r
\r
-UINT64\r
-PageAttributeToGcdAttribute (\r
- IN UINT64 PageAttributes\r
- )\r
-{\r
- UINT64 GcdAttributes;\r
-\r
- switch (PageAttributes & TT_ATTR_INDX_MASK) {\r
- case TT_ATTR_INDX_DEVICE_MEMORY:\r
- GcdAttributes = EFI_MEMORY_UC;\r
- break;\r
- case TT_ATTR_INDX_MEMORY_NON_CACHEABLE:\r
- GcdAttributes = EFI_MEMORY_WC;\r
- break;\r
- case TT_ATTR_INDX_MEMORY_WRITE_THROUGH:\r
- GcdAttributes = EFI_MEMORY_WT;\r
- break;\r
- case TT_ATTR_INDX_MEMORY_WRITE_BACK:\r
- GcdAttributes = EFI_MEMORY_WB;\r
- break;\r
- default:\r
- DEBUG ((EFI_D_ERROR, "PageAttributeToGcdAttribute: PageAttributes:0x%lX not supported.\n", PageAttributes));\r
- ASSERT (0);\r
- // The Global Coherency Domain (GCD) value is defined as a bit set.\r
- // Returning 0 means no attribute has been set.\r
- GcdAttributes = 0;\r
- }\r
-\r
- // Determine protection attributes\r
- if (((PageAttributes & TT_AP_MASK) == TT_AP_NO_RO) || ((PageAttributes & TT_AP_MASK) == TT_AP_RO_RO)) {\r
- // Read only cases map to write-protect\r
- GcdAttributes |= EFI_MEMORY_WP;\r
- }\r
-\r
- // Process eXecute Never attribute\r
- if ((PageAttributes & (TT_PXN_MASK | TT_UXN_MASK)) != 0 ) {\r
- GcdAttributes |= EFI_MEMORY_XP;\r
- }\r
-\r
- return GcdAttributes;\r
-}\r
-\r
-ARM_MEMORY_REGION_ATTRIBUTES\r
-GcdAttributeToArmAttribute (\r
- IN UINT64 GcdAttributes\r
- )\r
-{\r
- switch (GcdAttributes & 0xFF) {\r
- case EFI_MEMORY_UC:\r
- return ARM_MEMORY_REGION_ATTRIBUTE_DEVICE;\r
- case EFI_MEMORY_WC:\r
- return ARM_MEMORY_REGION_ATTRIBUTE_UNCACHED_UNBUFFERED;\r
- case EFI_MEMORY_WT:\r
- return ARM_MEMORY_REGION_ATTRIBUTE_WRITE_THROUGH;\r
- case EFI_MEMORY_WB:\r
- return ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK;\r
- default:\r
- DEBUG ((EFI_D_ERROR, "GcdAttributeToArmAttribute: 0x%lX attributes is not supported.\n", GcdAttributes));\r
- ASSERT (0);\r
- return ARM_MEMORY_REGION_ATTRIBUTE_DEVICE;\r
- }\r
-}\r
-\r
#define MIN_T0SZ 16\r
#define BITS_PER_LEVEL 9\r
\r
\r
STATIC\r
VOID\r
-ReplaceLiveEntry (\r
+ReplaceTableEntry (\r
IN UINT64 *Entry,\r
- IN UINT64 Value\r
+ IN UINT64 Value,\r
+ IN UINT64 RegionStart,\r
+ IN BOOLEAN IsLiveBlockMapping\r
)\r
{\r
- if (!ArmMmuEnabled ()) {\r
+ if (!ArmMmuEnabled () || !IsLiveBlockMapping) {\r
*Entry = Value;\r
+ ArmUpdateTranslationTableEntry (Entry, (VOID *)(UINTN)RegionStart);\r
} else {\r
- ArmReplaceLiveTranslationEntry (Entry, Value);\r
+ ArmReplaceLiveTranslationEntry (Entry, Value, RegionStart);\r
}\r
}\r
\r
STATIC\r
VOID\r
-LookupAddresstoRootTable (\r
- IN UINT64 MaxAddress,\r
- OUT UINTN *T0SZ,\r
- OUT UINTN *TableEntryCount\r
+FreePageTablesRecursive (\r
+ IN UINT64 *TranslationTable,\r
+ IN UINTN Level\r
)\r
{\r
- UINTN TopBit;\r
+ UINTN Index;\r
\r
- // Check the parameters are not NULL\r
- ASSERT ((T0SZ != NULL) && (TableEntryCount != NULL));\r
+ ASSERT (Level <= 3);\r
\r
- // Look for the highest bit set in MaxAddress\r
- for (TopBit = 63; TopBit != 0; TopBit--) {\r
- if ((1ULL << TopBit) & MaxAddress) {\r
- // MaxAddress top bit is found\r
- TopBit = TopBit + 1;\r
- break;\r
+ if (Level < 3) {\r
+ for (Index = 0; Index < TT_ENTRY_COUNT; Index++) {\r
+ if ((TranslationTable[Index] & TT_TYPE_MASK) == TT_TYPE_TABLE_ENTRY) {\r
+ FreePageTablesRecursive ((VOID *)(UINTN)(TranslationTable[Index] &\r
+ TT_ADDRESS_MASK_BLOCK_ENTRY),\r
+ Level + 1);\r
+ }\r
}\r
}\r
- ASSERT (TopBit != 0);\r
-\r
- // Calculate T0SZ from the top bit of the MaxAddress\r
- *T0SZ = 64 - TopBit;\r
-\r
- // Get the Table info from T0SZ\r
- GetRootTranslationTableInfo (*T0SZ, NULL, TableEntryCount);\r
+ FreePages (TranslationTable, 1);\r
}\r
\r
STATIC\r
-UINT64*\r
-GetBlockEntryListFromAddress (\r
- IN UINT64 *RootTable,\r
- IN UINT64 RegionStart,\r
- OUT UINTN *TableLevel,\r
- IN OUT UINT64 *BlockEntrySize,\r
- OUT UINT64 **LastBlockEntry\r
+BOOLEAN\r
+IsBlockEntry (\r
+ IN UINT64 Entry,\r
+ IN UINTN Level\r
)\r
{\r
- UINTN RootTableLevel;\r
- UINTN RootTableEntryCount;\r
- UINT64 *TranslationTable;\r
- UINT64 *BlockEntry;\r
- UINT64 *SubTableBlockEntry;\r
- UINT64 BlockEntryAddress;\r
- UINTN BaseAddressAlignment;\r
- UINTN PageLevel;\r
- UINTN Index;\r
- UINTN IndexLevel;\r
- UINTN T0SZ;\r
- UINT64 Attributes;\r
- UINT64 TableAttributes;\r
-\r
- // Initialize variable\r
- BlockEntry = NULL;\r
-\r
- // Ensure the parameters are valid\r
- if (!(TableLevel && BlockEntrySize && LastBlockEntry)) {\r
- ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER);\r
- return NULL;\r
- }\r
-\r
- // Ensure the Region is aligned on 4KB boundary\r
- if ((RegionStart & (SIZE_4KB - 1)) != 0) {\r
- ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER);\r
- return NULL;\r
+ if (Level == 3) {\r
+ return (Entry & TT_TYPE_MASK) == TT_TYPE_BLOCK_ENTRY_LEVEL3;\r
}\r
+ return (Entry & TT_TYPE_MASK) == TT_TYPE_BLOCK_ENTRY;\r
+}\r
\r
- // Ensure the required size is aligned on 4KB boundary and not 0\r
- if ((*BlockEntrySize & (SIZE_4KB - 1)) != 0 || *BlockEntrySize == 0) {\r
- ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER);\r
- return NULL;\r
- }\r
-\r
- T0SZ = ArmGetTCR () & TCR_T0SZ_MASK;\r
- // Get the Table info from T0SZ\r
- GetRootTranslationTableInfo (T0SZ, &RootTableLevel, &RootTableEntryCount);\r
-\r
- // If the start address is 0x0 then we use the size of the region to identify the alignment\r
- if (RegionStart == 0) {\r
- // Identify the highest possible alignment for the Region Size\r
- BaseAddressAlignment = LowBitSet64 (*BlockEntrySize);\r
- } else {\r
- // Identify the highest possible alignment for the Base Address\r
- BaseAddressAlignment = LowBitSet64 (RegionStart);\r
- }\r
-\r
- // Identify the Page Level the RegionStart must belong to. Note that PageLevel\r
- // should be at least 1 since block translations are not supported at level 0\r
- PageLevel = MAX (3 - ((BaseAddressAlignment - 12) / 9), 1);\r
-\r
- // If the required size is smaller than the current block size then we need to go to the page below.\r
- // The PageLevel was calculated on the Base Address alignment but did not take in account the alignment\r
- // of the allocation size\r
- while (*BlockEntrySize < TT_BLOCK_ENTRY_SIZE_AT_LEVEL (PageLevel)) {\r
- // It does not fit so we need to go a page level above\r
- PageLevel++;\r
+STATIC\r
+BOOLEAN\r
+IsTableEntry (\r
+ IN UINT64 Entry,\r
+ IN UINTN Level\r
+ )\r
+{\r
+ if (Level == 3) {\r
+ //\r
+ // TT_TYPE_TABLE_ENTRY aliases TT_TYPE_BLOCK_ENTRY_LEVEL3\r
+ // so we need to take the level into account as well.\r
+ //\r
+ return FALSE;\r
}\r
+ return (Entry & TT_TYPE_MASK) == TT_TYPE_TABLE_ENTRY;\r
+}\r
\r
- //\r
- // Get the Table Descriptor for the corresponding PageLevel. We need to decompose RegionStart to get appropriate entries\r
- //\r
-\r
- TranslationTable = RootTable;\r
- for (IndexLevel = RootTableLevel; IndexLevel <= PageLevel; IndexLevel++) {\r
- BlockEntry = (UINT64*)TT_GET_ENTRY_FOR_ADDRESS (TranslationTable, IndexLevel, RegionStart);\r
-\r
- if ((IndexLevel != 3) && ((*BlockEntry & TT_TYPE_MASK) == TT_TYPE_TABLE_ENTRY)) {\r
- // Go to the next table\r
- TranslationTable = (UINT64*)(*BlockEntry & TT_ADDRESS_MASK_DESCRIPTION_TABLE);\r
-\r
- // If we are at the last level then update the last level to next level\r
- if (IndexLevel == PageLevel) {\r
- // Enter the next level\r
- PageLevel++;\r
- }\r
- } else if ((*BlockEntry & TT_TYPE_MASK) == TT_TYPE_BLOCK_ENTRY) {\r
- // If we are not at the last level then we need to split this BlockEntry\r
- if (IndexLevel != PageLevel) {\r
- // Retrieve the attributes from the block entry\r
- Attributes = *BlockEntry & TT_ATTRIBUTES_MASK;\r
-\r
- // Convert the block entry attributes into Table descriptor attributes\r
- TableAttributes = TT_TABLE_AP_NO_PERMISSION;\r
- if (Attributes & TT_NS) {\r
- TableAttributes = TT_TABLE_NS;\r
+STATIC\r
+EFI_STATUS\r
+UpdateRegionMappingRecursive (\r
+ IN UINT64 RegionStart,\r
+ IN UINT64 RegionEnd,\r
+ IN UINT64 AttributeSetMask,\r
+ IN UINT64 AttributeClearMask,\r
+ IN UINT64 *PageTable,\r
+ IN UINTN Level\r
+ )\r
+{\r
+ UINTN BlockShift;\r
+ UINT64 BlockMask;\r
+ UINT64 BlockEnd;\r
+ UINT64 *Entry;\r
+ UINT64 EntryValue;\r
+ VOID *TranslationTable;\r
+ EFI_STATUS Status;\r
+\r
+ ASSERT (((RegionStart | RegionEnd) & EFI_PAGE_MASK) == 0);\r
+\r
+ BlockShift = (Level + 1) * BITS_PER_LEVEL + MIN_T0SZ;\r
+ BlockMask = MAX_UINT64 >> BlockShift;\r
+\r
+ DEBUG ((DEBUG_VERBOSE, "%a(%d): %llx - %llx set %lx clr %lx\n", __FUNCTION__,\r
+ Level, RegionStart, RegionEnd, AttributeSetMask, AttributeClearMask));\r
+\r
+ for (; RegionStart < RegionEnd; RegionStart = BlockEnd) {\r
+ BlockEnd = MIN (RegionEnd, (RegionStart | BlockMask) + 1);\r
+ Entry = &PageTable[(RegionStart >> (64 - BlockShift)) & (TT_ENTRY_COUNT - 1)];\r
+\r
+ //\r
+ // If RegionStart or BlockEnd is not aligned to the block size at this\r
+ // level, we will have to create a table mapping in order to map less\r
+ // than a block, and recurse to create the block or page entries at\r
+ // the next level. No block mappings are allowed at all at level 0,\r
+ // so in that case, we have to recurse unconditionally.\r
+ // If we are changing a table entry and the AttributeClearMask is non-zero,\r
+ // we cannot replace it with a block entry without potentially losing\r
+ // attribute information, so keep the table entry in that case.\r
+ //\r
+ if (Level == 0 || ((RegionStart | BlockEnd) & BlockMask) != 0 ||\r
+ (IsTableEntry (*Entry, Level) && AttributeClearMask != 0)) {\r
+ ASSERT (Level < 3);\r
+\r
+ if (!IsTableEntry (*Entry, Level)) {\r
+ //\r
+ // No table entry exists yet, so we need to allocate a page table\r
+ // for the next level.\r
+ //\r
+ TranslationTable = AllocatePages (1);\r
+ if (TranslationTable == NULL) {\r
+ return EFI_OUT_OF_RESOURCES;\r
}\r
\r
- // Get the address corresponding at this entry\r
- BlockEntryAddress = RegionStart;\r
- BlockEntryAddress = BlockEntryAddress >> TT_ADDRESS_OFFSET_AT_LEVEL(IndexLevel);\r
- // Shift back to right to set zero before the effective address\r
- BlockEntryAddress = BlockEntryAddress << TT_ADDRESS_OFFSET_AT_LEVEL(IndexLevel);\r
-\r
- // Set the correct entry type for the next page level\r
- if ((IndexLevel + 1) == 3) {\r
- Attributes |= TT_TYPE_BLOCK_ENTRY_LEVEL3;\r
- } else {\r
- Attributes |= TT_TYPE_BLOCK_ENTRY;\r
+ if (!ArmMmuEnabled ()) {\r
+ //\r
+ // Make sure we are not inadvertently hitting in the caches\r
+ // when populating the page tables.\r
+ //\r
+ InvalidateDataCacheRange (TranslationTable, EFI_PAGE_SIZE);\r
}\r
\r
- // Create a new translation table\r
- TranslationTable = (UINT64*)AllocateAlignedPages (EFI_SIZE_TO_PAGES(TT_ENTRY_COUNT * sizeof(UINT64)), TT_ALIGNMENT_DESCRIPTION_TABLE);\r
- if (TranslationTable == NULL) {\r
- return NULL;\r
+ ZeroMem (TranslationTable, EFI_PAGE_SIZE);\r
+\r
+ if (IsBlockEntry (*Entry, Level)) {\r
+ //\r
+ // We are splitting an existing block entry, so we have to populate\r
+ // the new table with the attributes of the block entry it replaces.\r
+ //\r
+ Status = UpdateRegionMappingRecursive (RegionStart & ~BlockMask,\r
+ (RegionStart | BlockMask) + 1, *Entry & TT_ATTRIBUTES_MASK,\r
+ 0, TranslationTable, Level + 1);\r
+ if (EFI_ERROR (Status)) {\r
+ //\r
+ // The range we passed to UpdateRegionMappingRecursive () is block\r
+ // aligned, so it is guaranteed that no further pages were allocated\r
+ // by it, and so we only have to free the page we allocated here.\r
+ //\r
+ FreePages (TranslationTable, 1);\r
+ return Status;\r
+ }\r
}\r
+ } else {\r
+ TranslationTable = (VOID *)(UINTN)(*Entry & TT_ADDRESS_MASK_BLOCK_ENTRY);\r
+ }\r
\r
- // Populate the newly created lower level table\r
- SubTableBlockEntry = TranslationTable;\r
- for (Index = 0; Index < TT_ENTRY_COUNT; Index++) {\r
- *SubTableBlockEntry = Attributes | (BlockEntryAddress + (Index << TT_ADDRESS_OFFSET_AT_LEVEL(IndexLevel + 1)));\r
- SubTableBlockEntry++;\r
+ //\r
+ // Recurse to the next level\r
+ //\r
+ Status = UpdateRegionMappingRecursive (RegionStart, BlockEnd,\r
+ AttributeSetMask, AttributeClearMask, TranslationTable,\r
+ Level + 1);\r
+ if (EFI_ERROR (Status)) {\r
+ if (!IsTableEntry (*Entry, Level)) {\r
+ //\r
+ // We are creating a new table entry, so on failure, we can free all\r
+ // allocations we made recursively, given that the whole subhierarchy\r
+ // has not been wired into the live page tables yet. (This is not\r
+ // possible for existing table entries, since we cannot revert the\r
+ // modifications we made to the subhierarchy it represents.)\r
+ //\r
+ FreePageTablesRecursive (TranslationTable, Level + 1);\r
}\r
+ return Status;\r
+ }\r
\r
- // Fill the BlockEntry with the new TranslationTable\r
- ReplaceLiveEntry (BlockEntry,\r
- ((UINTN)TranslationTable & TT_ADDRESS_MASK_DESCRIPTION_TABLE) | TableAttributes | TT_TYPE_TABLE_ENTRY);\r
+ if (!IsTableEntry (*Entry, Level)) {\r
+ EntryValue = (UINTN)TranslationTable | TT_TYPE_TABLE_ENTRY;\r
+ ReplaceTableEntry (Entry, EntryValue, RegionStart,\r
+ IsBlockEntry (*Entry, Level));\r
}\r
} else {\r
- if (IndexLevel != PageLevel) {\r
+ EntryValue = (*Entry & AttributeClearMask) | AttributeSetMask;\r
+ EntryValue |= RegionStart;\r
+ EntryValue |= (Level == 3) ? TT_TYPE_BLOCK_ENTRY_LEVEL3\r
+ : TT_TYPE_BLOCK_ENTRY;\r
+\r
+ if (IsTableEntry (*Entry, Level)) {\r
//\r
- // Case when we have an Invalid Entry and we are at a page level above of the one targetted.\r
+ // We are replacing a table entry with a block entry. This is only\r
+ // possible if we are keeping none of the original attributes.\r
+ // We can free the table entry's page table, and all the ones below\r
+ // it, since we are dropping the only possible reference to it.\r
//\r
-\r
- // Create a new translation table\r
- TranslationTable = (UINT64*)AllocateAlignedPages (EFI_SIZE_TO_PAGES(TT_ENTRY_COUNT * sizeof(UINT64)), TT_ALIGNMENT_DESCRIPTION_TABLE);\r
- if (TranslationTable == NULL) {\r
- return NULL;\r
- }\r
-\r
- ZeroMem (TranslationTable, TT_ENTRY_COUNT * sizeof(UINT64));\r
-\r
- // Fill the new BlockEntry with the TranslationTable\r
- *BlockEntry = ((UINTN)TranslationTable & TT_ADDRESS_MASK_DESCRIPTION_TABLE) | TT_TYPE_TABLE_ENTRY;\r
+ ASSERT (AttributeClearMask == 0);\r
+ TranslationTable = (VOID *)(UINTN)(*Entry & TT_ADDRESS_MASK_BLOCK_ENTRY);\r
+ ReplaceTableEntry (Entry, EntryValue, RegionStart, TRUE);\r
+ FreePageTablesRecursive (TranslationTable, Level + 1);\r
+ } else {\r
+ ReplaceTableEntry (Entry, EntryValue, RegionStart, FALSE);\r
}\r
}\r
}\r
+ return EFI_SUCCESS;\r
+}\r
\r
- // Expose the found PageLevel to the caller\r
- *TableLevel = PageLevel;\r
+STATIC\r
+VOID\r
+LookupAddresstoRootTable (\r
+ IN UINT64 MaxAddress,\r
+ OUT UINTN *T0SZ,\r
+ OUT UINTN *TableEntryCount\r
+ )\r
+{\r
+ UINTN TopBit;\r
\r
- // Now, we have the Table Level we can get the Block Size associated to this table\r
- *BlockEntrySize = TT_BLOCK_ENTRY_SIZE_AT_LEVEL (PageLevel);\r
+ // Check the parameters are not NULL\r
+ ASSERT ((T0SZ != NULL) && (TableEntryCount != NULL));\r
\r
- // The last block of the root table depends on the number of entry in this table,\r
- // otherwise it is always the (TT_ENTRY_COUNT - 1)th entry in the table.\r
- *LastBlockEntry = TT_LAST_BLOCK_ADDRESS(TranslationTable,\r
- (PageLevel == RootTableLevel) ? RootTableEntryCount : TT_ENTRY_COUNT);\r
+ // Look for the highest bit set in MaxAddress\r
+ for (TopBit = 63; TopBit != 0; TopBit--) {\r
+ if ((1ULL << TopBit) & MaxAddress) {\r
+ // MaxAddress top bit is found\r
+ TopBit = TopBit + 1;\r
+ break;\r
+ }\r
+ }\r
+ ASSERT (TopBit != 0);\r
+\r
+ // Calculate T0SZ from the top bit of the MaxAddress\r
+ *T0SZ = 64 - TopBit;\r
\r
- return BlockEntry;\r
+ // Get the Table info from T0SZ\r
+ GetRootTranslationTableInfo (*T0SZ, NULL, TableEntryCount);\r
}\r
\r
STATIC\r
-RETURN_STATUS\r
+EFI_STATUS\r
UpdateRegionMapping (\r
- IN UINT64 *RootTable,\r
IN UINT64 RegionStart,\r
IN UINT64 RegionLength,\r
- IN UINT64 Attributes,\r
- IN UINT64 BlockEntryMask\r
+ IN UINT64 AttributeSetMask,\r
+ IN UINT64 AttributeClearMask\r
)\r
{\r
- UINT32 Type;\r
- UINT64 *BlockEntry;\r
- UINT64 *LastBlockEntry;\r
- UINT64 BlockEntrySize;\r
- UINTN TableLevel;\r
-\r
- // Ensure the Length is aligned on 4KB boundary\r
- if ((RegionLength == 0) || ((RegionLength & (SIZE_4KB - 1)) != 0)) {\r
- ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER);\r
- return RETURN_INVALID_PARAMETER;\r
- }\r
+ UINTN RootTableLevel;\r
+ UINTN T0SZ;\r
\r
- do {\r
- // Get the first Block Entry that matches the Virtual Address and also the information on the Table Descriptor\r
- // such as the the size of the Block Entry and the address of the last BlockEntry of the Table Descriptor\r
- BlockEntrySize = RegionLength;\r
- BlockEntry = GetBlockEntryListFromAddress (RootTable, RegionStart, &TableLevel, &BlockEntrySize, &LastBlockEntry);\r
- if (BlockEntry == NULL) {\r
- // GetBlockEntryListFromAddress() return NULL when it fails to allocate new pages from the Translation Tables\r
- return RETURN_OUT_OF_RESOURCES;\r
- }\r
-\r
- if (TableLevel != 3) {\r
- Type = TT_TYPE_BLOCK_ENTRY;\r
- } else {\r
- Type = TT_TYPE_BLOCK_ENTRY_LEVEL3;\r
- }\r
+ if (((RegionStart | RegionLength) & EFI_PAGE_MASK)) {\r
+ return EFI_INVALID_PARAMETER;\r
+ }\r
\r
- do {\r
- // Fill the Block Entry with attribute and output block address\r
- *BlockEntry &= BlockEntryMask;\r
- *BlockEntry |= (RegionStart & TT_ADDRESS_MASK_BLOCK_ENTRY) | Attributes | Type;\r
-\r
- // Go to the next BlockEntry\r
- RegionStart += BlockEntrySize;\r
- RegionLength -= BlockEntrySize;\r
- BlockEntry++;\r
-\r
- // Break the inner loop when next block is a table\r
- // Rerun GetBlockEntryListFromAddress to avoid page table memory leak\r
- if (TableLevel != 3 &&\r
- (*BlockEntry & TT_TYPE_MASK) == TT_TYPE_TABLE_ENTRY) {\r
- break;\r
- }\r
- } while ((RegionLength >= BlockEntrySize) && (BlockEntry <= LastBlockEntry));\r
- } while (RegionLength != 0);\r
+ T0SZ = ArmGetTCR () & TCR_T0SZ_MASK;\r
+ GetRootTranslationTableInfo (T0SZ, &RootTableLevel, NULL);\r
\r
- return RETURN_SUCCESS;\r
+ return UpdateRegionMappingRecursive (RegionStart, RegionStart + RegionLength,\r
+ AttributeSetMask, AttributeClearMask, ArmGetTTBR0BaseAddress (),\r
+ RootTableLevel);\r
}\r
\r
STATIC\r
-RETURN_STATUS\r
+EFI_STATUS\r
FillTranslationTable (\r
IN UINT64 *RootTable,\r
IN ARM_MEMORY_REGION_DESCRIPTOR *MemoryRegion\r
)\r
{\r
return UpdateRegionMapping (\r
- RootTable,\r
MemoryRegion->VirtualBase,\r
MemoryRegion->Length,\r
ArmMemoryAttributeToPageAttribute (MemoryRegion->Attributes) | TT_AF,\r
);\r
}\r
\r
-RETURN_STATUS\r
-SetMemoryAttributes (\r
- IN EFI_PHYSICAL_ADDRESS BaseAddress,\r
- IN UINT64 Length,\r
- IN UINT64 Attributes,\r
- IN EFI_PHYSICAL_ADDRESS VirtualMask\r
+STATIC\r
+UINT64\r
+GcdAttributeToPageAttribute (\r
+ IN UINT64 GcdAttributes\r
)\r
{\r
- RETURN_STATUS Status;\r
- ARM_MEMORY_REGION_DESCRIPTOR MemoryRegion;\r
- UINT64 *TranslationTable;\r
+ UINT64 PageAttributes;\r
\r
- MemoryRegion.PhysicalBase = BaseAddress;\r
- MemoryRegion.VirtualBase = BaseAddress;\r
- MemoryRegion.Length = Length;\r
- MemoryRegion.Attributes = GcdAttributeToArmAttribute (Attributes);\r
+ switch (GcdAttributes & EFI_MEMORY_CACHETYPE_MASK) {\r
+ case EFI_MEMORY_UC:\r
+ PageAttributes = TT_ATTR_INDX_DEVICE_MEMORY;\r
+ break;\r
+ case EFI_MEMORY_WC:\r
+ PageAttributes = TT_ATTR_INDX_MEMORY_NON_CACHEABLE;\r
+ break;\r
+ case EFI_MEMORY_WT:\r
+ PageAttributes = TT_ATTR_INDX_MEMORY_WRITE_THROUGH | TT_SH_INNER_SHAREABLE;\r
+ break;\r
+ case EFI_MEMORY_WB:\r
+ PageAttributes = TT_ATTR_INDX_MEMORY_WRITE_BACK | TT_SH_INNER_SHAREABLE;\r
+ break;\r
+ default:\r
+ PageAttributes = TT_ATTR_INDX_MASK;\r
+ break;\r
+ }\r
\r
- TranslationTable = ArmGetTTBR0BaseAddress ();\r
+ if ((GcdAttributes & EFI_MEMORY_XP) != 0 ||\r
+ (GcdAttributes & EFI_MEMORY_CACHETYPE_MASK) == EFI_MEMORY_UC) {\r
+ if (ArmReadCurrentEL () == AARCH64_EL2) {\r
+ PageAttributes |= TT_XN_MASK;\r
+ } else {\r
+ PageAttributes |= TT_UXN_MASK | TT_PXN_MASK;\r
+ }\r
+ }\r
\r
- Status = FillTranslationTable (TranslationTable, &MemoryRegion);\r
- if (RETURN_ERROR (Status)) {\r
- return Status;\r
+ if ((GcdAttributes & EFI_MEMORY_RO) != 0) {\r
+ PageAttributes |= TT_AP_RO_RO;\r
}\r
\r
- // Invalidate all TLB entries so changes are synced\r
- ArmInvalidateTlb ();\r
+ return PageAttributes | TT_AF;\r
+}\r
\r
- return RETURN_SUCCESS;\r
+EFI_STATUS\r
+ArmSetMemoryAttributes (\r
+ IN EFI_PHYSICAL_ADDRESS BaseAddress,\r
+ IN UINT64 Length,\r
+ IN UINT64 Attributes\r
+ )\r
+{\r
+ UINT64 PageAttributes;\r
+ UINT64 PageAttributeMask;\r
+\r
+ PageAttributes = GcdAttributeToPageAttribute (Attributes);\r
+ PageAttributeMask = 0;\r
+\r
+ if ((Attributes & EFI_MEMORY_CACHETYPE_MASK) == 0) {\r
+ //\r
+ // No memory type was set in Attributes, so we are going to update the\r
+ // permissions only.\r
+ //\r
+ PageAttributes &= TT_AP_MASK | TT_UXN_MASK | TT_PXN_MASK;\r
+ PageAttributeMask = ~(TT_ADDRESS_MASK_BLOCK_ENTRY | TT_AP_MASK |\r
+ TT_PXN_MASK | TT_XN_MASK);\r
+ }\r
+\r
+ return UpdateRegionMapping (BaseAddress, Length, PageAttributes,\r
+ PageAttributeMask);\r
}\r
\r
STATIC\r
-RETURN_STATUS\r
+EFI_STATUS\r
SetMemoryRegionAttribute (\r
IN EFI_PHYSICAL_ADDRESS BaseAddress,\r
IN UINT64 Length,\r
IN UINT64 BlockEntryMask\r
)\r
{\r
- RETURN_STATUS Status;\r
- UINT64 *RootTable;\r
-\r
- RootTable = ArmGetTTBR0BaseAddress ();\r
-\r
- Status = UpdateRegionMapping (RootTable, BaseAddress, Length, Attributes, BlockEntryMask);\r
- if (RETURN_ERROR (Status)) {\r
- return Status;\r
- }\r
-\r
- // Invalidate all TLB entries so changes are synced\r
- ArmInvalidateTlb ();\r
-\r
- return RETURN_SUCCESS;\r
+ return UpdateRegionMapping (BaseAddress, Length, Attributes, BlockEntryMask);\r
}\r
\r
-RETURN_STATUS\r
+EFI_STATUS\r
ArmSetMemoryRegionNoExec (\r
IN EFI_PHYSICAL_ADDRESS BaseAddress,\r
IN UINT64 Length\r
~TT_ADDRESS_MASK_BLOCK_ENTRY);\r
}\r
\r
-RETURN_STATUS\r
+EFI_STATUS\r
ArmClearMemoryRegionNoExec (\r
IN EFI_PHYSICAL_ADDRESS BaseAddress,\r
IN UINT64 Length\r
Mask);\r
}\r
\r
-RETURN_STATUS\r
+EFI_STATUS\r
ArmSetMemoryRegionReadOnly (\r
IN EFI_PHYSICAL_ADDRESS BaseAddress,\r
IN UINT64 Length\r
~TT_ADDRESS_MASK_BLOCK_ENTRY);\r
}\r
\r
-RETURN_STATUS\r
+EFI_STATUS\r
ArmClearMemoryRegionReadOnly (\r
IN EFI_PHYSICAL_ADDRESS BaseAddress,\r
IN UINT64 Length\r
~(TT_ADDRESS_MASK_BLOCK_ENTRY | TT_AP_MASK));\r
}\r
\r
-RETURN_STATUS\r
+EFI_STATUS\r
EFIAPI\r
ArmConfigureMmu (\r
IN ARM_MEMORY_REGION_DESCRIPTOR *MemoryTable,\r
)\r
{\r
VOID* TranslationTable;\r
- UINTN TranslationTablePageCount;\r
- UINT32 TranslationTableAttribute;\r
- ARM_MEMORY_REGION_DESCRIPTOR *MemoryTableEntry;\r
UINT64 MaxAddress;\r
- UINT64 TopAddress;\r
UINTN T0SZ;\r
UINTN RootTableEntryCount;\r
UINT64 TCR;\r
- RETURN_STATUS Status;\r
+ EFI_STATUS Status;\r
\r
- if(MemoryTable == NULL) {\r
+ if (MemoryTable == NULL) {\r
ASSERT (MemoryTable != NULL);\r
- return RETURN_INVALID_PARAMETER;\r
+ return EFI_INVALID_PARAMETER;\r
}\r
\r
- // Identify the highest address of the memory table\r
- MaxAddress = MemoryTable->PhysicalBase + MemoryTable->Length - 1;\r
- MemoryTableEntry = MemoryTable;\r
- while (MemoryTableEntry->Length != 0) {\r
- TopAddress = MemoryTableEntry->PhysicalBase + MemoryTableEntry->Length - 1;\r
- if (TopAddress > MaxAddress) {\r
- MaxAddress = TopAddress;\r
- }\r
- MemoryTableEntry++;\r
- }\r
+ //\r
+ // Limit the virtual address space to what we can actually use: UEFI\r
+ // mandates a 1:1 mapping, so no point in making the virtual address\r
+ // space larger than the physical address space. We also have to take\r
+ // into account the architectural limitations that result from UEFI's\r
+ // use of 4 KB pages.\r
+ //\r
+ MaxAddress = MIN (LShiftU64 (1ULL, ArmGetPhysicalAddressBits ()) - 1,\r
+ MAX_ALLOC_ADDRESS);\r
\r
// Lookup the Table Level to get the information\r
LookupAddresstoRootTable (MaxAddress, &T0SZ, &RootTableEntryCount);\r
} else if (MaxAddress < SIZE_256TB) {\r
TCR |= TCR_PS_256TB;\r
} else {\r
- DEBUG ((EFI_D_ERROR, "ArmConfigureMmu: The MaxAddress 0x%lX is not supported by this MMU configuration.\n", MaxAddress));\r
+ DEBUG ((DEBUG_ERROR,\r
+ "ArmConfigureMmu: The MaxAddress 0x%lX is not supported by this MMU configuration.\n",\r
+ MaxAddress));\r
ASSERT (0); // Bigger than 48-bit memory space are not supported\r
- return RETURN_UNSUPPORTED;\r
+ return EFI_UNSUPPORTED;\r
}\r
} else if (ArmReadCurrentEL () == AARCH64_EL1) {\r
// Due to Cortex-A57 erratum #822227 we must set TG1[1] == 1, regardless of EPD1.\r
} else if (MaxAddress < SIZE_256TB) {\r
TCR |= TCR_IPS_256TB;\r
} else {\r
- DEBUG ((EFI_D_ERROR, "ArmConfigureMmu: The MaxAddress 0x%lX is not supported by this MMU configuration.\n", MaxAddress));\r
+ DEBUG ((DEBUG_ERROR,\r
+ "ArmConfigureMmu: The MaxAddress 0x%lX is not supported by this MMU configuration.\n",\r
+ MaxAddress));\r
ASSERT (0); // Bigger than 48-bit memory space are not supported\r
- return RETURN_UNSUPPORTED;\r
+ return EFI_UNSUPPORTED;\r
}\r
} else {\r
ASSERT (0); // UEFI is only expected to run at EL2 and EL1, not EL3.\r
- return RETURN_UNSUPPORTED;\r
+ return EFI_UNSUPPORTED;\r
}\r
\r
+ //\r
+ // Translation table walks are always cache coherent on ARMv8-A, so cache\r
+ // maintenance on page tables is never needed. Since there is a risk of\r
+ // loss of coherency when using mismatched attributes, and given that memory\r
+ // is mapped cacheable except for extraordinary cases (such as non-coherent\r
+ // DMA), have the page table walker perform cached accesses as well, and\r
+ // assert below that that matches the attributes we use for CPU accesses to\r
+ // the region.\r
+ //\r
+ TCR |= TCR_SH_INNER_SHAREABLE |\r
+ TCR_RGN_OUTER_WRITE_BACK_ALLOC |\r
+ TCR_RGN_INNER_WRITE_BACK_ALLOC;\r
+\r
// Set TCR\r
ArmSetTCR (TCR);\r
\r
// Allocate pages for translation table\r
- TranslationTablePageCount = EFI_SIZE_TO_PAGES(RootTableEntryCount * sizeof(UINT64));\r
- TranslationTable = (UINT64*)AllocateAlignedPages (TranslationTablePageCount, TT_ALIGNMENT_DESCRIPTION_TABLE);\r
+ TranslationTable = AllocatePages (1);\r
if (TranslationTable == NULL) {\r
- return RETURN_OUT_OF_RESOURCES;\r
+ return EFI_OUT_OF_RESOURCES;\r
}\r
- // We set TTBR0 just after allocating the table to retrieve its location from the subsequent\r
- // functions without needing to pass this value across the functions. The MMU is only enabled\r
- // after the translation tables are populated.\r
+ //\r
+ // We set TTBR0 just after allocating the table to retrieve its location from\r
+ // the subsequent functions without needing to pass this value across the\r
+ // functions. The MMU is only enabled after the translation tables are\r
+ // populated.\r
+ //\r
ArmSetTTBR0 (TranslationTable);\r
\r
if (TranslationTableBase != NULL) {\r
}\r
\r
if (TranslationTableSize != NULL) {\r
- *TranslationTableSize = RootTableEntryCount * sizeof(UINT64);\r
+ *TranslationTableSize = RootTableEntryCount * sizeof (UINT64);\r
}\r
\r
- ZeroMem (TranslationTable, RootTableEntryCount * sizeof(UINT64));\r
-\r
- // Disable MMU and caches. ArmDisableMmu() also invalidates the TLBs\r
- ArmDisableMmu ();\r
- ArmDisableDataCache ();\r
- ArmDisableInstructionCache ();\r
-\r
- // Make sure nothing sneaked into the cache\r
- ArmCleanInvalidateDataCache ();\r
- ArmInvalidateInstructionCache ();\r
+ //\r
+ // Make sure we are not inadvertently hitting in the caches\r
+ // when populating the page tables.\r
+ //\r
+ InvalidateDataCacheRange (TranslationTable,\r
+ RootTableEntryCount * sizeof (UINT64));\r
+ ZeroMem (TranslationTable, RootTableEntryCount * sizeof (UINT64));\r
\r
- TranslationTableAttribute = TT_ATTR_INDX_INVALID;\r
while (MemoryTable->Length != 0) {\r
- // Find the memory attribute for the Translation Table\r
- if (((UINTN)TranslationTable >= MemoryTable->PhysicalBase) &&\r
- ((UINTN)TranslationTable <= MemoryTable->PhysicalBase - 1 + MemoryTable->Length)) {\r
- TranslationTableAttribute = MemoryTable->Attributes;\r
- }\r
-\r
Status = FillTranslationTable (TranslationTable, MemoryTable);\r
- if (RETURN_ERROR (Status)) {\r
- goto FREE_TRANSLATION_TABLE;\r
+ if (EFI_ERROR (Status)) {\r
+ goto FreeTranslationTable;\r
}\r
MemoryTable++;\r
}\r
\r
- // Translate the Memory Attributes into Translation Table Register Attributes\r
- if ((TranslationTableAttribute == ARM_MEMORY_REGION_ATTRIBUTE_UNCACHED_UNBUFFERED) ||\r
- (TranslationTableAttribute == ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_UNCACHED_UNBUFFERED)) {\r
- TCR |= TCR_SH_NON_SHAREABLE | TCR_RGN_OUTER_NON_CACHEABLE | TCR_RGN_INNER_NON_CACHEABLE;\r
- } else if ((TranslationTableAttribute == ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK) ||\r
- (TranslationTableAttribute == ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK)) {\r
- TCR |= TCR_SH_INNER_SHAREABLE | TCR_RGN_OUTER_WRITE_BACK_ALLOC | TCR_RGN_INNER_WRITE_BACK_ALLOC;\r
- } else if ((TranslationTableAttribute == ARM_MEMORY_REGION_ATTRIBUTE_WRITE_THROUGH) ||\r
- (TranslationTableAttribute == ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_THROUGH)) {\r
- TCR |= TCR_SH_NON_SHAREABLE | TCR_RGN_OUTER_WRITE_THROUGH | TCR_RGN_INNER_WRITE_THROUGH;\r
- } else {\r
- // If we failed to find a mapping that contains the root translation table then it probably means the translation table\r
- // is not mapped in the given memory map.\r
- ASSERT (0);\r
- Status = RETURN_UNSUPPORTED;\r
- goto FREE_TRANSLATION_TABLE;\r
- }\r
-\r
- // Set again TCR after getting the Translation Table attributes\r
- ArmSetTCR (TCR);\r
-\r
- ArmSetMAIR (MAIR_ATTR(TT_ATTR_INDX_DEVICE_MEMORY, MAIR_ATTR_DEVICE_MEMORY) | // mapped to EFI_MEMORY_UC\r
- MAIR_ATTR(TT_ATTR_INDX_MEMORY_NON_CACHEABLE, MAIR_ATTR_NORMAL_MEMORY_NON_CACHEABLE) | // mapped to EFI_MEMORY_WC\r
- MAIR_ATTR(TT_ATTR_INDX_MEMORY_WRITE_THROUGH, MAIR_ATTR_NORMAL_MEMORY_WRITE_THROUGH) | // mapped to EFI_MEMORY_WT\r
- MAIR_ATTR(TT_ATTR_INDX_MEMORY_WRITE_BACK, MAIR_ATTR_NORMAL_MEMORY_WRITE_BACK)); // mapped to EFI_MEMORY_WB\r
+ //\r
+ // EFI_MEMORY_UC ==> MAIR_ATTR_DEVICE_MEMORY\r
+ // EFI_MEMORY_WC ==> MAIR_ATTR_NORMAL_MEMORY_NON_CACHEABLE\r
+ // EFI_MEMORY_WT ==> MAIR_ATTR_NORMAL_MEMORY_WRITE_THROUGH\r
+ // EFI_MEMORY_WB ==> MAIR_ATTR_NORMAL_MEMORY_WRITE_BACK\r
+ //\r
+ ArmSetMAIR (\r
+ MAIR_ATTR (TT_ATTR_INDX_DEVICE_MEMORY, MAIR_ATTR_DEVICE_MEMORY) |\r
+ MAIR_ATTR (TT_ATTR_INDX_MEMORY_NON_CACHEABLE, MAIR_ATTR_NORMAL_MEMORY_NON_CACHEABLE) |\r
+ MAIR_ATTR (TT_ATTR_INDX_MEMORY_WRITE_THROUGH, MAIR_ATTR_NORMAL_MEMORY_WRITE_THROUGH) |\r
+ MAIR_ATTR (TT_ATTR_INDX_MEMORY_WRITE_BACK, MAIR_ATTR_NORMAL_MEMORY_WRITE_BACK)\r
+ );\r
\r
ArmDisableAlignmentCheck ();\r
+ ArmEnableStackAlignmentCheck ();\r
ArmEnableInstructionCache ();\r
ArmEnableDataCache ();\r
\r
ArmEnableMmu ();\r
- return RETURN_SUCCESS;\r
+ return EFI_SUCCESS;\r
\r
-FREE_TRANSLATION_TABLE:\r
- FreePages (TranslationTable, TranslationTablePageCount);\r
+FreeTranslationTable:\r
+ FreePages (TranslationTable, 1);\r
return Status;\r
}\r
\r
projects / mirror_edk2.git / blobdiff