ArmPkg/Drivers/CpuDxe/AArch64/Mmu.c

   1 /*++
   2
   3 Copyright (c) 2009, Hewlett-Packard Company. All rights reserved.<BR>
   4 Portions copyright (c) 2010, Apple Inc. All rights reserved.<BR>
   5 Portions copyright (c) 2011-2021, Arm Limited. All rights reserved.<BR>
   6 Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
   7
   8 SPDX-License-Identifier: BSD-2-Clause-Patent
   9
  10
  11 --*/
  12
  13 #include <Library/MemoryAllocationLib.h>
  14 #include "CpuDxe.h"
  15
  16 #define INVALID_ENTRY  ((UINT32)~0)
  17
  18 #define MIN_T0SZ        16
  19 #define BITS_PER_LEVEL  9
  20
  21 STATIC
  22 VOID
  23 GetRootTranslationTableInfo (
  24   IN  UINTN  T0SZ,
  25   OUT UINTN  *RootTableLevel,
  26   OUT UINTN  *RootTableEntryCount
  27   )
  28 {
  29   *RootTableLevel      = (T0SZ - MIN_T0SZ) / BITS_PER_LEVEL;
  30   *RootTableEntryCount = TT_ENTRY_COUNT >> (T0SZ - MIN_T0SZ) % BITS_PER_LEVEL;
  31 }
  32
  33 STATIC
  34 UINT64
  35 PageAttributeToGcdAttribute (
  36   IN UINT64  PageAttributes
  37   )
  38 {
  39   UINT64  GcdAttributes;
  40
  41   switch (PageAttributes & TT_ATTR_INDX_MASK) {
  42     case TT_ATTR_INDX_DEVICE_MEMORY:
  43       GcdAttributes = EFI_MEMORY_UC;
  44       break;
  45     case TT_ATTR_INDX_MEMORY_NON_CACHEABLE:
  46       GcdAttributes = EFI_MEMORY_WC;
  47       break;
  48     case TT_ATTR_INDX_MEMORY_WRITE_THROUGH:
  49       GcdAttributes = EFI_MEMORY_WT;
  50       break;
  51     case TT_ATTR_INDX_MEMORY_WRITE_BACK:
  52       GcdAttributes = EFI_MEMORY_WB;
  53       break;
  54     default:
  55       DEBUG ((
  56         DEBUG_ERROR,
  57         "PageAttributeToGcdAttribute: PageAttributes:0x%lX not supported.\n",
  58         PageAttributes
  59         ));
  60       ASSERT (0);
  61       // The Global Coherency Domain (GCD) value is defined as a bit set.
  62       // Returning 0 means no attribute has been set.
  63       GcdAttributes = 0;
  64   }
  65
  66   // Determine protection attributes
  67   if (((PageAttributes & TT_AP_MASK) == TT_AP_NO_RO) ||
  68       ((PageAttributes & TT_AP_MASK) == TT_AP_RO_RO))
  69   {
  70     // Read only cases map to write-protect
  71     GcdAttributes |= EFI_MEMORY_RO;
  72   }
  73
  74   // Process eXecute Never attribute
  75   if ((PageAttributes & (TT_PXN_MASK | TT_UXN_MASK)) != 0) {
  76     GcdAttributes |= EFI_MEMORY_XP;
  77   }
  78
  79   return GcdAttributes;
  80 }
  81
  82 STATIC
  83 UINT64
  84 GetFirstPageAttribute (
  85   IN UINT64  *FirstLevelTableAddress,
  86   IN UINTN   TableLevel
  87   )
  88 {
  89   UINT64  FirstEntry;
  90
  91   // Get the first entry of the table
  92   FirstEntry = *FirstLevelTableAddress;
  93
  94   if ((TableLevel != 3) && ((FirstEntry & TT_TYPE_MASK) == TT_TYPE_TABLE_ENTRY)) {
  95     // Only valid for Levels 0, 1 and 2
  96
  97     // Get the attribute of the subsequent table
  98     return GetFirstPageAttribute ((UINT64 *)(FirstEntry & TT_ADDRESS_MASK_DESCRIPTION_TABLE), TableLevel + 1);
  99   } else if (((FirstEntry & TT_TYPE_MASK) == TT_TYPE_BLOCK_ENTRY) ||
 100              ((TableLevel == 3) && ((FirstEntry & TT_TYPE_MASK) == TT_TYPE_BLOCK_ENTRY_LEVEL3)))
 101   {
 102     return FirstEntry & TT_ATTR_INDX_MASK;
 103   } else {
 104     return INVALID_ENTRY;
 105   }
 106 }
 107
 108 STATIC
 109 UINT64
 110 GetNextEntryAttribute (
 111   IN     UINT64  *TableAddress,
 112   IN     UINTN   EntryCount,
 113   IN     UINTN   TableLevel,
 114   IN     UINT64  BaseAddress,
 115   IN OUT UINT32  *PrevEntryAttribute,
 116   IN OUT UINT64  *StartGcdRegion
 117   )
 118 {
 119   UINTN                            Index;
 120   UINT64                           Entry;
 121   UINT32                           EntryAttribute;
 122   UINT32                           EntryType;
 123   EFI_STATUS                       Status;
 124   UINTN                            NumberOfDescriptors;
 125   EFI_GCD_MEMORY_SPACE_DESCRIPTOR  *MemorySpaceMap;
 126
 127   // Get the memory space map from GCD
 128   MemorySpaceMap = NULL;
 129   Status         = gDS->GetMemorySpaceMap (&NumberOfDescriptors, &MemorySpaceMap);
 130   ASSERT_EFI_ERROR (Status);
 131
 132   // We cannot get more than 3-level page table
 133   ASSERT (TableLevel <= 3);
 134
 135   // While the top level table might not contain TT_ENTRY_COUNT entries;
 136   // the subsequent ones should be filled up
 137   for (Index = 0; Index < EntryCount; Index++) {
 138     Entry          = TableAddress[Index];
 139     EntryType      = Entry & TT_TYPE_MASK;
 140     EntryAttribute = Entry  & TT_ATTR_INDX_MASK;
 141
 142     // If Entry is a Table Descriptor type entry then go through the sub-level table
 143     if ((EntryType == TT_TYPE_BLOCK_ENTRY) ||
 144         ((TableLevel == 3) && (EntryType == TT_TYPE_BLOCK_ENTRY_LEVEL3)))
 145     {
 146       if ((*PrevEntryAttribute == INVALID_ENTRY) || (EntryAttribute != *PrevEntryAttribute)) {
 147         if (*PrevEntryAttribute != INVALID_ENTRY) {
 148           // Update GCD with the last region
 149           SetGcdMemorySpaceAttributes (
 150             MemorySpaceMap,
 151             NumberOfDescriptors,
 152             *StartGcdRegion,
 153             (BaseAddress + (Index * TT_ADDRESS_AT_LEVEL (TableLevel))) - *StartGcdRegion,
 154             PageAttributeToGcdAttribute (*PrevEntryAttribute)
 155             );
 156         }
 157
 158         // Start of the new region
 159         *StartGcdRegion     = BaseAddress + (Index * TT_ADDRESS_AT_LEVEL (TableLevel));
 160         *PrevEntryAttribute = EntryAttribute;
 161       } else {
 162         continue;
 163       }
 164     } else if (EntryType == TT_TYPE_TABLE_ENTRY) {
 165       // Table Entry type is only valid for Level 0, 1, 2
 166       ASSERT (TableLevel < 3);
 167
 168       // Increase the level number and scan the sub-level table
 169       GetNextEntryAttribute (
 170         (UINT64 *)(Entry & TT_ADDRESS_MASK_DESCRIPTION_TABLE),
 171         TT_ENTRY_COUNT,
 172         TableLevel + 1,
 173         (BaseAddress + (Index * TT_ADDRESS_AT_LEVEL (TableLevel))),
 174         PrevEntryAttribute,
 175         StartGcdRegion
 176         );
 177     } else {
 178       if (*PrevEntryAttribute != INVALID_ENTRY) {
 179         // Update GCD with the last region
 180         SetGcdMemorySpaceAttributes (
 181           MemorySpaceMap,
 182           NumberOfDescriptors,
 183           *StartGcdRegion,
 184           (BaseAddress + (Index * TT_ADDRESS_AT_LEVEL (TableLevel))) - *StartGcdRegion,
 185           PageAttributeToGcdAttribute (*PrevEntryAttribute)
 186           );
 187
 188         // Start of the new region
 189         *StartGcdRegion     = BaseAddress + (Index * TT_ADDRESS_AT_LEVEL (TableLevel));
 190         *PrevEntryAttribute = INVALID_ENTRY;
 191       }
 192     }
 193   }
 194
 195   FreePool (MemorySpaceMap);
 196
 197   return BaseAddress + (EntryCount * TT_ADDRESS_AT_LEVEL (TableLevel));
 198 }
 199
 200 EFI_STATUS
 201 SyncCacheConfig (
 202   IN  EFI_CPU_ARCH_PROTOCOL  *CpuProtocol
 203   )
 204 {
 205   EFI_STATUS                       Status;
 206   UINT32                           PageAttribute;
 207   UINT64                           *FirstLevelTableAddress;
 208   UINTN                            TableLevel;
 209   UINTN                            TableCount;
 210   UINTN                            NumberOfDescriptors;
 211   EFI_GCD_MEMORY_SPACE_DESCRIPTOR  *MemorySpaceMap;
 212   UINTN                            Tcr;
 213   UINTN                            T0SZ;
 214   UINT64                           BaseAddressGcdRegion;
 215   UINT64                           EndAddressGcdRegion;
 216
 217   // This code assumes MMU is enabled and filed with section translations
 218   ASSERT (ArmMmuEnabled ());
 219
 220   //
 221   // Get the memory space map from GCD
 222   //
 223   MemorySpaceMap = NULL;
 224   Status         = gDS->GetMemorySpaceMap (&NumberOfDescriptors, &MemorySpaceMap);
 225   ASSERT_EFI_ERROR (Status);
 226
 227   // The GCD implementation maintains its own copy of the state of memory space attributes.  GCD needs
 228   // to know what the initial memory space attributes are.  The CPU Arch. Protocol does not provide a
 229   // GetMemoryAttributes function for GCD to get this so we must resort to calling GCD (as if we were
 230   // a client) to update its copy of the attributes.  This is bad architecture and should be replaced
 231   // with a way for GCD to query the CPU Arch. driver of the existing memory space attributes instead.
 232
 233   // Obtain page table base
 234   FirstLevelTableAddress = (UINT64 *)(ArmGetTTBR0BaseAddress ());
 235
 236   // Get Translation Control Register value
 237   Tcr = ArmGetTCR ();
 238   // Get Address Region Size
 239   T0SZ = Tcr & TCR_T0SZ_MASK;
 240
 241   // Get the level of the first table for the indicated Address Region Size
 242   GetRootTranslationTableInfo (T0SZ, &TableLevel, &TableCount);
 243
 244   // First Attribute of the Page Tables
 245   PageAttribute = GetFirstPageAttribute (FirstLevelTableAddress, TableLevel);
 246
 247   // We scan from the start of the memory map (ie: at the address 0x0)
 248   BaseAddressGcdRegion = 0x0;
 249   EndAddressGcdRegion  = GetNextEntryAttribute (
 250                            FirstLevelTableAddress,
 251                            TableCount,
 252                            TableLevel,
 253                            BaseAddressGcdRegion,
 254                            &PageAttribute,
 255                            &BaseAddressGcdRegion
 256                            );
 257
 258   // Update GCD with the last region if valid
 259   if (PageAttribute != INVALID_ENTRY) {
 260     SetGcdMemorySpaceAttributes (
 261       MemorySpaceMap,
 262       NumberOfDescriptors,
 263       BaseAddressGcdRegion,
 264       EndAddressGcdRegion - BaseAddressGcdRegion,
 265       PageAttributeToGcdAttribute (PageAttribute)
 266       );
 267   }
 268
 269   FreePool (MemorySpaceMap);
 270
 271   return EFI_SUCCESS;
 272 }
 273
 274 UINT64
 275 EfiAttributeToArmAttribute (
 276   IN UINT64  EfiAttributes
 277   )
 278 {
 279   UINT64  ArmAttributes;
 280
 281   switch (EfiAttributes & EFI_MEMORY_CACHETYPE_MASK) {
 282     case EFI_MEMORY_UC:
 283       if (ArmReadCurrentEL () == AARCH64_EL2) {
 284         ArmAttributes = TT_ATTR_INDX_DEVICE_MEMORY | TT_XN_MASK;
 285       } else {
 286         ArmAttributes = TT_ATTR_INDX_DEVICE_MEMORY | TT_UXN_MASK | TT_PXN_MASK;
 287       }
 288
 289       break;
 290     case EFI_MEMORY_WC:
 291       ArmAttributes = TT_ATTR_INDX_MEMORY_NON_CACHEABLE;
 292       break;
 293     case EFI_MEMORY_WT:
 294       ArmAttributes = TT_ATTR_INDX_MEMORY_WRITE_THROUGH | TT_SH_INNER_SHAREABLE;
 295       break;
 296     case EFI_MEMORY_WB:
 297       ArmAttributes = TT_ATTR_INDX_MEMORY_WRITE_BACK | TT_SH_INNER_SHAREABLE;
 298       break;
 299     default:
 300       ArmAttributes = TT_ATTR_INDX_MASK;
 301   }
 302
 303   // Set the access flag to match the block attributes
 304   ArmAttributes |= TT_AF;
 305
 306   // Determine protection attributes
 307   if ((EfiAttributes & EFI_MEMORY_RO) != 0) {
 308     ArmAttributes |= TT_AP_RO_RO;
 309   }
 310
 311   // Process eXecute Never attribute
 312   if ((EfiAttributes & EFI_MEMORY_XP) != 0) {
 313     ArmAttributes |= TT_PXN_MASK;
 314   }
 315
 316   return ArmAttributes;
 317 }
 318
 319 // This function will recursively go down the page table to find the first block address linked to 'BaseAddress'.
 320 // And then the function will identify the size of the region that has the same page table attribute.
 321 EFI_STATUS
 322 GetMemoryRegionRec (
 323   IN     UINT64  *TranslationTable,
 324   IN     UINTN   TableLevel,
 325   IN     UINT64  *LastBlockEntry,
 326   IN OUT UINTN   *BaseAddress,
 327   OUT    UINTN   *RegionLength,
 328   OUT    UINTN   *RegionAttributes
 329   )
 330 {
 331   EFI_STATUS  Status;
 332   UINT64      *NextTranslationTable;
 333   UINT64      *BlockEntry;
 334   UINT64      BlockEntryType;
 335   UINT64      EntryType;
 336
 337   if (TableLevel != 3) {
 338     BlockEntryType = TT_TYPE_BLOCK_ENTRY;
 339   } else {
 340     BlockEntryType = TT_TYPE_BLOCK_ENTRY_LEVEL3;
 341   }
 342
 343   // Find the block entry linked to the Base Address
 344   BlockEntry = (UINT64 *)TT_GET_ENTRY_FOR_ADDRESS (TranslationTable, TableLevel, *BaseAddress);
 345   EntryType  = *BlockEntry & TT_TYPE_MASK;
 346
 347   if ((TableLevel < 3) && (EntryType == TT_TYPE_TABLE_ENTRY)) {
 348     NextTranslationTable = (UINT64 *)(*BlockEntry & TT_ADDRESS_MASK_DESCRIPTION_TABLE);
 349
 350     // The entry is a page table, so we go to the next level
 351     Status = GetMemoryRegionRec (
 352                NextTranslationTable, // Address of the next level page table
 353                TableLevel + 1,       // Next Page Table level
 354                (UINTN *)TT_LAST_BLOCK_ADDRESS (NextTranslationTable, TT_ENTRY_COUNT),
 355                BaseAddress,
 356                RegionLength,
 357                RegionAttributes
 358                );
 359
 360     // In case of 'Success', it means the end of the block region has been found into the upper
 361     // level translation table
 362     if (!EFI_ERROR (Status)) {
 363       return EFI_SUCCESS;
 364     }
 365
 366     // Now we processed the table move to the next entry
 367     BlockEntry++;
 368   } else if (EntryType == BlockEntryType) {
 369     // We have found the BlockEntry attached to the address. We save its start address (the start
 370     // address might be before the 'BaseAddress') and attributes
 371     *BaseAddress      = *BaseAddress & ~(TT_ADDRESS_AT_LEVEL (TableLevel) - 1);
 372     *RegionLength     = 0;
 373     *RegionAttributes = *BlockEntry & TT_ATTRIBUTES_MASK;
 374   } else {
 375     // We have an 'Invalid' entry
 376     return EFI_UNSUPPORTED;
 377   }
 378
 379   while (BlockEntry <= LastBlockEntry) {
 380     if ((*BlockEntry & TT_ATTRIBUTES_MASK) == *RegionAttributes) {
 381       *RegionLength = *RegionLength + TT_BLOCK_ENTRY_SIZE_AT_LEVEL (TableLevel);
 382     } else {
 383       // In case we have found the end of the region we return success
 384       return EFI_SUCCESS;
 385     }
 386
 387     BlockEntry++;
 388   }
 389
 390   // If we have reached the end of the TranslationTable and we have not found the end of the region then
 391   // we return EFI_NOT_FOUND.
 392   // The caller will continue to look for the memory region at its level
 393   return EFI_NOT_FOUND;
 394 }
 395
 396 EFI_STATUS
 397 GetMemoryRegion (
 398   IN OUT UINTN  *BaseAddress,
 399   OUT    UINTN  *RegionLength,
 400   OUT    UINTN  *RegionAttributes
 401   )
 402 {
 403   EFI_STATUS  Status;
 404   UINT64      *TranslationTable;
 405   UINTN       TableLevel;
 406   UINTN       EntryCount;
 407   UINTN       T0SZ;
 408
 409   ASSERT ((BaseAddress != NULL) && (RegionLength != NULL) && (RegionAttributes != NULL));
 410
 411   TranslationTable = ArmGetTTBR0BaseAddress ();
 412
 413   T0SZ = ArmGetTCR () & TCR_T0SZ_MASK;
 414   // Get the Table info from T0SZ
 415   GetRootTranslationTableInfo (T0SZ, &TableLevel, &EntryCount);
 416
 417   Status = GetMemoryRegionRec (
 418              TranslationTable,
 419              TableLevel,
 420              (UINTN *)TT_LAST_BLOCK_ADDRESS (TranslationTable, EntryCount),
 421              BaseAddress,
 422              RegionLength,
 423              RegionAttributes
 424              );
 425
 426   // If the region continues up to the end of the root table then GetMemoryRegionRec()
 427   // will return EFI_NOT_FOUND
 428   if (Status == EFI_NOT_FOUND) {
 429     return EFI_SUCCESS;
 430   } else {
 431     return Status;
 432   }
 433 }