ArmPkg/Drivers/CpuDxe/AArch64/Mmu.c

   1 /*++
   2
   3 Copyright (c) 2009, Hewlett-Packard Company. All rights reserved.<BR>
   4 Portions copyright (c) 2010, Apple Inc. All rights reserved.<BR>
   5 Portions copyright (c) 2011-2021, Arm Limited. All rights reserved.<BR>
   6 Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
   7
   8 SPDX-License-Identifier: BSD-2-Clause-Patent
   9
  10
  11 --*/
  12
  13 #include <Library/MemoryAllocationLib.h>
  14 #include "CpuDxe.h"
  15
  16 #define INVALID_ENTRY   ((UINT32)~0)
  17
  18 #define MIN_T0SZ        16
  19 #define BITS_PER_LEVEL  9
  20
  21 STATIC
  22 VOID
  23 GetRootTranslationTableInfo (
  24   IN  UINTN     T0SZ,
  25   OUT UINTN     *RootTableLevel,
  26   OUT UINTN     *RootTableEntryCount
  27   )
  28 {
  29   *RootTableLevel       = (T0SZ - MIN_T0SZ) / BITS_PER_LEVEL;
  30   *RootTableEntryCount  = TT_ENTRY_COUNT >> (T0SZ - MIN_T0SZ) % BITS_PER_LEVEL;
  31 }
  32
  33 STATIC
  34 UINT64
  35 PageAttributeToGcdAttribute (
  36   IN UINT64 PageAttributes
  37   )
  38 {
  39   UINT64  GcdAttributes;
  40
  41   switch (PageAttributes & TT_ATTR_INDX_MASK) {
  42   case TT_ATTR_INDX_DEVICE_MEMORY:
  43     GcdAttributes = EFI_MEMORY_UC;
  44     break;
  45   case TT_ATTR_INDX_MEMORY_NON_CACHEABLE:
  46     GcdAttributes = EFI_MEMORY_WC;
  47     break;
  48   case TT_ATTR_INDX_MEMORY_WRITE_THROUGH:
  49     GcdAttributes = EFI_MEMORY_WT;
  50     break;
  51   case TT_ATTR_INDX_MEMORY_WRITE_BACK:
  52     GcdAttributes = EFI_MEMORY_WB;
  53     break;
  54   default:
  55     DEBUG ((DEBUG_ERROR,
  56       "PageAttributeToGcdAttribute: PageAttributes:0x%lX not supported.\n",
  57       PageAttributes));
  58     ASSERT (0);
  59     // The Global Coherency Domain (GCD) value is defined as a bit set.
  60     // Returning 0 means no attribute has been set.
  61     GcdAttributes = 0;
  62   }
  63
  64   // Determine protection attributes
  65   if (((PageAttributes & TT_AP_MASK) == TT_AP_NO_RO) ||
  66       ((PageAttributes & TT_AP_MASK) == TT_AP_RO_RO)) {
  67     // Read only cases map to write-protect
  68     GcdAttributes |= EFI_MEMORY_RO;
  69   }
  70
  71   // Process eXecute Never attribute
  72   if ((PageAttributes & (TT_PXN_MASK | TT_UXN_MASK)) != 0) {
  73     GcdAttributes |= EFI_MEMORY_XP;
  74   }
  75
  76   return GcdAttributes;
  77 }
  78
  79 STATIC
  80 UINT64
  81 GetFirstPageAttribute (
  82   IN UINT64  *FirstLevelTableAddress,
  83   IN UINTN    TableLevel
  84   )
  85 {
  86   UINT64 FirstEntry;
  87
  88   // Get the first entry of the table
  89   FirstEntry = *FirstLevelTableAddress;
  90
  91   if ((TableLevel != 3) && (FirstEntry & TT_TYPE_MASK) == TT_TYPE_TABLE_ENTRY) {
  92     // Only valid for Levels 0, 1 and 2
  93
  94     // Get the attribute of the subsequent table
  95     return GetFirstPageAttribute ((UINT64*)(FirstEntry & TT_ADDRESS_MASK_DESCRIPTION_TABLE), TableLevel + 1);
  96   } else if (((FirstEntry & TT_TYPE_MASK) == TT_TYPE_BLOCK_ENTRY) ||
  97              ((TableLevel == 3) && ((FirstEntry & TT_TYPE_MASK) == TT_TYPE_BLOCK_ENTRY_LEVEL3)))
  98   {
  99     return FirstEntry & TT_ATTR_INDX_MASK;
 100   } else {
 101     return INVALID_ENTRY;
 102   }
 103 }
 104
 105 STATIC
 106 UINT64
 107 GetNextEntryAttribute (
 108   IN     UINT64 *TableAddress,
 109   IN     UINTN   EntryCount,
 110   IN     UINTN   TableLevel,
 111   IN     UINT64  BaseAddress,
 112   IN OUT UINT32 *PrevEntryAttribute,
 113   IN OUT UINT64 *StartGcdRegion
 114   )
 115 {
 116   UINTN                             Index;
 117   UINT64                            Entry;
 118   UINT32                            EntryAttribute;
 119   UINT32                            EntryType;
 120   EFI_STATUS                        Status;
 121   UINTN                             NumberOfDescriptors;
 122   EFI_GCD_MEMORY_SPACE_DESCRIPTOR  *MemorySpaceMap;
 123
 124   // Get the memory space map from GCD
 125   MemorySpaceMap = NULL;
 126   Status = gDS->GetMemorySpaceMap (&NumberOfDescriptors, &MemorySpaceMap);
 127   ASSERT_EFI_ERROR (Status);
 128
 129   // We cannot get more than 3-level page table
 130   ASSERT (TableLevel <= 3);
 131
 132   // While the top level table might not contain TT_ENTRY_COUNT entries;
 133   // the subsequent ones should be filled up
 134   for (Index = 0; Index < EntryCount; Index++) {
 135     Entry = TableAddress[Index];
 136     EntryType = Entry & TT_TYPE_MASK;
 137     EntryAttribute = Entry  & TT_ATTR_INDX_MASK;
 138
 139     // If Entry is a Table Descriptor type entry then go through the sub-level table
 140     if ((EntryType == TT_TYPE_BLOCK_ENTRY) ||
 141         ((TableLevel == 3) && (EntryType == TT_TYPE_BLOCK_ENTRY_LEVEL3))) {
 142       if ((*PrevEntryAttribute == INVALID_ENTRY) || (EntryAttribute != *PrevEntryAttribute)) {
 143         if (*PrevEntryAttribute != INVALID_ENTRY) {
 144           // Update GCD with the last region
 145           SetGcdMemorySpaceAttributes (MemorySpaceMap, NumberOfDescriptors,
 146               *StartGcdRegion,
 147               (BaseAddress + (Index * TT_ADDRESS_AT_LEVEL(TableLevel))) - *StartGcdRegion,
 148               PageAttributeToGcdAttribute (*PrevEntryAttribute));
 149         }
 150
 151         // Start of the new region
 152         *StartGcdRegion = BaseAddress + (Index * TT_ADDRESS_AT_LEVEL(TableLevel));
 153         *PrevEntryAttribute = EntryAttribute;
 154       } else {
 155         continue;
 156       }
 157     } else if (EntryType == TT_TYPE_TABLE_ENTRY) {
 158       // Table Entry type is only valid for Level 0, 1, 2
 159       ASSERT (TableLevel < 3);
 160
 161       // Increase the level number and scan the sub-level table
 162       GetNextEntryAttribute ((UINT64*)(Entry & TT_ADDRESS_MASK_DESCRIPTION_TABLE),
 163                              TT_ENTRY_COUNT, TableLevel + 1,
 164                              (BaseAddress + (Index * TT_ADDRESS_AT_LEVEL(TableLevel))),
 165                              PrevEntryAttribute, StartGcdRegion);
 166     } else {
 167       if (*PrevEntryAttribute != INVALID_ENTRY) {
 168         // Update GCD with the last region
 169         SetGcdMemorySpaceAttributes (MemorySpaceMap, NumberOfDescriptors,
 170             *StartGcdRegion,
 171             (BaseAddress + (Index * TT_ADDRESS_AT_LEVEL(TableLevel))) - *StartGcdRegion,
 172             PageAttributeToGcdAttribute (*PrevEntryAttribute));
 173
 174         // Start of the new region
 175         *StartGcdRegion = BaseAddress + (Index * TT_ADDRESS_AT_LEVEL(TableLevel));
 176         *PrevEntryAttribute = INVALID_ENTRY;
 177       }
 178     }
 179   }
 180
 181   FreePool (MemorySpaceMap);
 182
 183   return BaseAddress + (EntryCount * TT_ADDRESS_AT_LEVEL(TableLevel));
 184 }
 185
 186 EFI_STATUS
 187 SyncCacheConfig (
 188   IN  EFI_CPU_ARCH_PROTOCOL *CpuProtocol
 189   )
 190 {
 191   EFI_STATUS                          Status;
 192   UINT32                              PageAttribute;
 193   UINT64                             *FirstLevelTableAddress;
 194   UINTN                               TableLevel;
 195   UINTN                               TableCount;
 196   UINTN                               NumberOfDescriptors;
 197   EFI_GCD_MEMORY_SPACE_DESCRIPTOR    *MemorySpaceMap;
 198   UINTN                               Tcr;
 199   UINTN                               T0SZ;
 200   UINT64                              BaseAddressGcdRegion;
 201   UINT64                              EndAddressGcdRegion;
 202
 203   // This code assumes MMU is enabled and filed with section translations
 204   ASSERT (ArmMmuEnabled ());
 205
 206   //
 207   // Get the memory space map from GCD
 208   //
 209   MemorySpaceMap = NULL;
 210   Status = gDS->GetMemorySpaceMap (&NumberOfDescriptors, &MemorySpaceMap);
 211   ASSERT_EFI_ERROR (Status);
 212
 213   // The GCD implementation maintains its own copy of the state of memory space attributes.  GCD needs
 214   // to know what the initial memory space attributes are.  The CPU Arch. Protocol does not provide a
 215   // GetMemoryAttributes function for GCD to get this so we must resort to calling GCD (as if we were
 216   // a client) to update its copy of the attributes.  This is bad architecture and should be replaced
 217   // with a way for GCD to query the CPU Arch. driver of the existing memory space attributes instead.
 218
 219   // Obtain page table base
 220   FirstLevelTableAddress = (UINT64*)(ArmGetTTBR0BaseAddress ());
 221
 222   // Get Translation Control Register value
 223   Tcr = ArmGetTCR ();
 224   // Get Address Region Size
 225   T0SZ = Tcr & TCR_T0SZ_MASK;
 226
 227   // Get the level of the first table for the indicated Address Region Size
 228   GetRootTranslationTableInfo (T0SZ, &TableLevel, &TableCount);
 229
 230   // First Attribute of the Page Tables
 231   PageAttribute = GetFirstPageAttribute (FirstLevelTableAddress, TableLevel);
 232
 233   // We scan from the start of the memory map (ie: at the address 0x0)
 234   BaseAddressGcdRegion = 0x0;
 235   EndAddressGcdRegion = GetNextEntryAttribute (FirstLevelTableAddress,
 236                                                TableCount, TableLevel,
 237                                                BaseAddressGcdRegion,
 238                                                &PageAttribute, &BaseAddressGcdRegion);
 239
 240   // Update GCD with the last region if valid
 241   if (PageAttribute != INVALID_ENTRY) {
 242     SetGcdMemorySpaceAttributes (MemorySpaceMap, NumberOfDescriptors,
 243         BaseAddressGcdRegion,
 244         EndAddressGcdRegion - BaseAddressGcdRegion,
 245         PageAttributeToGcdAttribute (PageAttribute));
 246   }
 247
 248   FreePool (MemorySpaceMap);
 249
 250   return EFI_SUCCESS;
 251 }
 252
 253 UINT64
 254 EfiAttributeToArmAttribute (
 255   IN UINT64                    EfiAttributes
 256   )
 257 {
 258   UINT64 ArmAttributes;
 259
 260   switch (EfiAttributes & EFI_MEMORY_CACHETYPE_MASK) {
 261   case EFI_MEMORY_UC:
 262     if (ArmReadCurrentEL () == AARCH64_EL2) {
 263       ArmAttributes = TT_ATTR_INDX_DEVICE_MEMORY | TT_XN_MASK;
 264     } else {
 265       ArmAttributes = TT_ATTR_INDX_DEVICE_MEMORY | TT_UXN_MASK | TT_PXN_MASK;
 266     }
 267     break;
 268   case EFI_MEMORY_WC:
 269     ArmAttributes = TT_ATTR_INDX_MEMORY_NON_CACHEABLE;
 270     break;
 271   case EFI_MEMORY_WT:
 272     ArmAttributes = TT_ATTR_INDX_MEMORY_WRITE_THROUGH | TT_SH_INNER_SHAREABLE;
 273     break;
 274   case EFI_MEMORY_WB:
 275     ArmAttributes = TT_ATTR_INDX_MEMORY_WRITE_BACK | TT_SH_INNER_SHAREABLE;
 276     break;
 277   default:
 278     ArmAttributes = TT_ATTR_INDX_MASK;
 279   }
 280
 281   // Set the access flag to match the block attributes
 282   ArmAttributes |= TT_AF;
 283
 284   // Determine protection attributes
 285   if ((EfiAttributes & EFI_MEMORY_RO) != 0) {
 286     ArmAttributes |= TT_AP_RO_RO;
 287   }
 288
 289   // Process eXecute Never attribute
 290   if ((EfiAttributes & EFI_MEMORY_XP) != 0) {
 291     ArmAttributes |= TT_PXN_MASK;
 292   }
 293
 294   return ArmAttributes;
 295 }
 296
 297 // This function will recursively go down the page table to find the first block address linked to 'BaseAddress'.
 298 // And then the function will identify the size of the region that has the same page table attribute.
 299 EFI_STATUS
 300 GetMemoryRegionRec (
 301   IN     UINT64                  *TranslationTable,
 302   IN     UINTN                    TableLevel,
 303   IN     UINT64                  *LastBlockEntry,
 304   IN OUT UINTN                   *BaseAddress,
 305   OUT    UINTN                   *RegionLength,
 306   OUT    UINTN                   *RegionAttributes
 307   )
 308 {
 309   EFI_STATUS Status;
 310   UINT64    *NextTranslationTable;
 311   UINT64    *BlockEntry;
 312   UINT64     BlockEntryType;
 313   UINT64     EntryType;
 314
 315   if (TableLevel != 3) {
 316     BlockEntryType = TT_TYPE_BLOCK_ENTRY;
 317   } else {
 318     BlockEntryType = TT_TYPE_BLOCK_ENTRY_LEVEL3;
 319   }
 320
 321   // Find the block entry linked to the Base Address
 322   BlockEntry = (UINT64*)TT_GET_ENTRY_FOR_ADDRESS (TranslationTable, TableLevel, *BaseAddress);
 323   EntryType = *BlockEntry & TT_TYPE_MASK;
 324
 325   if ((TableLevel < 3) && (EntryType == TT_TYPE_TABLE_ENTRY)) {
 326     NextTranslationTable = (UINT64*)(*BlockEntry & TT_ADDRESS_MASK_DESCRIPTION_TABLE);
 327
 328     // The entry is a page table, so we go to the next level
 329     Status = GetMemoryRegionRec (
 330         NextTranslationTable, // Address of the next level page table
 331         TableLevel + 1, // Next Page Table level
 332         (UINTN*)TT_LAST_BLOCK_ADDRESS(NextTranslationTable, TT_ENTRY_COUNT),
 333         BaseAddress, RegionLength, RegionAttributes);
 334
 335     // In case of 'Success', it means the end of the block region has been found into the upper
 336     // level translation table
 337     if (!EFI_ERROR(Status)) {
 338       return EFI_SUCCESS;
 339     }
 340
 341     // Now we processed the table move to the next entry
 342     BlockEntry++;
 343   } else if (EntryType == BlockEntryType) {
 344     // We have found the BlockEntry attached to the address. We save its start address (the start
 345     // address might be before the 'BaseAddress') and attributes
 346     *BaseAddress      = *BaseAddress & ~(TT_ADDRESS_AT_LEVEL(TableLevel) - 1);
 347     *RegionLength     = 0;
 348     *RegionAttributes = *BlockEntry & TT_ATTRIBUTES_MASK;
 349   } else {
 350     // We have an 'Invalid' entry
 351     return EFI_UNSUPPORTED;
 352   }
 353
 354   while (BlockEntry <= LastBlockEntry) {
 355     if ((*BlockEntry & TT_ATTRIBUTES_MASK) == *RegionAttributes) {
 356       *RegionLength = *RegionLength + TT_BLOCK_ENTRY_SIZE_AT_LEVEL(TableLevel);
 357     } else {
 358       // In case we have found the end of the region we return success
 359       return EFI_SUCCESS;
 360     }
 361     BlockEntry++;
 362   }
 363
 364   // If we have reached the end of the TranslationTable and we have not found the end of the region then
 365   // we return EFI_NOT_FOUND.
 366   // The caller will continue to look for the memory region at its level
 367   return EFI_NOT_FOUND;
 368 }
 369
 370 EFI_STATUS
 371 GetMemoryRegion (
 372   IN OUT UINTN                   *BaseAddress,
 373   OUT    UINTN                   *RegionLength,
 374   OUT    UINTN                   *RegionAttributes
 375   )
 376 {
 377   EFI_STATUS  Status;
 378   UINT64     *TranslationTable;
 379   UINTN       TableLevel;
 380   UINTN       EntryCount;
 381   UINTN       T0SZ;
 382
 383   ASSERT ((BaseAddress != NULL) && (RegionLength != NULL) && (RegionAttributes != NULL));
 384
 385   TranslationTable = ArmGetTTBR0BaseAddress ();
 386
 387   T0SZ = ArmGetTCR () & TCR_T0SZ_MASK;
 388   // Get the Table info from T0SZ
 389   GetRootTranslationTableInfo (T0SZ, &TableLevel, &EntryCount);
 390
 391   Status = GetMemoryRegionRec (TranslationTable, TableLevel,
 392       (UINTN*)TT_LAST_BLOCK_ADDRESS(TranslationTable, EntryCount),
 393       BaseAddress, RegionLength, RegionAttributes);
 394
 395   // If the region continues up to the end of the root table then GetMemoryRegionRec()
 396   // will return EFI_NOT_FOUND
 397   if (Status == EFI_NOT_FOUND) {
 398     return EFI_SUCCESS;
 399   } else {
 400     return Status;
 401   }
 402 }