2 * File managing the MMU for ARMv8 architecture
4 * Copyright (c) 2011-2020, ARM Limited. All rights reserved.
5 * Copyright (c) 2016, Linaro Limited. All rights reserved.
6 * Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
8 * SPDX-License-Identifier: BSD-2-Clause-Patent
13 #include <Chipset/AArch64.h>
14 #include <Library/BaseMemoryLib.h>
15 #include <Library/CacheMaintenanceLib.h>
16 #include <Library/MemoryAllocationLib.h>
17 #include <Library/ArmLib.h>
18 #include <Library/ArmMmuLib.h>
19 #include <Library/BaseLib.h>
20 #include <Library/DebugLib.h>
24 ArmMemoryAttributeToPageAttribute (
25 IN ARM_MEMORY_REGION_ATTRIBUTES Attributes
29 case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK_NONSHAREABLE
:
30 case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK_NONSHAREABLE
:
31 return TT_ATTR_INDX_MEMORY_WRITE_BACK
;
33 case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK
:
34 case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK
:
35 return TT_ATTR_INDX_MEMORY_WRITE_BACK
| TT_SH_INNER_SHAREABLE
;
37 case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_THROUGH
:
38 case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_THROUGH
:
39 return TT_ATTR_INDX_MEMORY_WRITE_THROUGH
| TT_SH_INNER_SHAREABLE
;
41 // Uncached and device mappings are treated as outer shareable by default,
42 case ARM_MEMORY_REGION_ATTRIBUTE_UNCACHED_UNBUFFERED
:
43 case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_UNCACHED_UNBUFFERED
:
44 return TT_ATTR_INDX_MEMORY_NON_CACHEABLE
;
48 case ARM_MEMORY_REGION_ATTRIBUTE_DEVICE
:
49 case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_DEVICE
:
50 if (ArmReadCurrentEL () == AARCH64_EL2
) {
51 return TT_ATTR_INDX_DEVICE_MEMORY
| TT_XN_MASK
;
53 return TT_ATTR_INDX_DEVICE_MEMORY
| TT_UXN_MASK
| TT_PXN_MASK
;
59 #define BITS_PER_LEVEL 9
60 #define MAX_VA_BITS 48
64 GetRootTableEntryCount (
68 return TT_ENTRY_COUNT
>> (T0SZ
- MIN_T0SZ
) % BITS_PER_LEVEL
;
77 return (T0SZ
- MIN_T0SZ
) / BITS_PER_LEVEL
;
85 IN UINT64 RegionStart
,
86 IN BOOLEAN IsLiveBlockMapping
89 if (!ArmMmuEnabled () || !IsLiveBlockMapping
) {
91 ArmUpdateTranslationTableEntry (Entry
, (VOID
*)(UINTN
)RegionStart
);
93 ArmReplaceLiveTranslationEntry (Entry
, Value
, RegionStart
);
99 FreePageTablesRecursive (
100 IN UINT64
*TranslationTable
,
109 for (Index
= 0; Index
< TT_ENTRY_COUNT
; Index
++) {
110 if ((TranslationTable
[Index
] & TT_TYPE_MASK
) == TT_TYPE_TABLE_ENTRY
) {
111 FreePageTablesRecursive (
112 (VOID
*)(UINTN
)(TranslationTable
[Index
] &
113 TT_ADDRESS_MASK_BLOCK_ENTRY
),
120 FreePages (TranslationTable
, 1);
131 return (Entry
& TT_TYPE_MASK
) == TT_TYPE_BLOCK_ENTRY_LEVEL3
;
134 return (Entry
& TT_TYPE_MASK
) == TT_TYPE_BLOCK_ENTRY
;
146 // TT_TYPE_TABLE_ENTRY aliases TT_TYPE_BLOCK_ENTRY_LEVEL3
147 // so we need to take the level into account as well.
152 return (Entry
& TT_TYPE_MASK
) == TT_TYPE_TABLE_ENTRY
;
157 UpdateRegionMappingRecursive (
158 IN UINT64 RegionStart
,
160 IN UINT64 AttributeSetMask
,
161 IN UINT64 AttributeClearMask
,
162 IN UINT64
*PageTable
,
171 VOID
*TranslationTable
;
174 ASSERT (((RegionStart
| RegionEnd
) & EFI_PAGE_MASK
) == 0);
176 BlockShift
= (Level
+ 1) * BITS_PER_LEVEL
+ MIN_T0SZ
;
177 BlockMask
= MAX_UINT64
>> BlockShift
;
181 "%a(%d): %llx - %llx set %lx clr %lx\n",
190 for ( ; RegionStart
< RegionEnd
; RegionStart
= BlockEnd
) {
191 BlockEnd
= MIN (RegionEnd
, (RegionStart
| BlockMask
) + 1);
192 Entry
= &PageTable
[(RegionStart
>> (64 - BlockShift
)) & (TT_ENTRY_COUNT
- 1)];
195 // If RegionStart or BlockEnd is not aligned to the block size at this
196 // level, we will have to create a table mapping in order to map less
197 // than a block, and recurse to create the block or page entries at
198 // the next level. No block mappings are allowed at all at level 0,
199 // so in that case, we have to recurse unconditionally.
200 // If we are changing a table entry and the AttributeClearMask is non-zero,
201 // we cannot replace it with a block entry without potentially losing
202 // attribute information, so keep the table entry in that case.
204 if ((Level
== 0) || (((RegionStart
| BlockEnd
) & BlockMask
) != 0) ||
205 (IsTableEntry (*Entry
, Level
) && (AttributeClearMask
!= 0)))
209 if (!IsTableEntry (*Entry
, Level
)) {
211 // No table entry exists yet, so we need to allocate a page table
212 // for the next level.
214 TranslationTable
= AllocatePages (1);
215 if (TranslationTable
== NULL
) {
216 return EFI_OUT_OF_RESOURCES
;
219 if (!ArmMmuEnabled ()) {
221 // Make sure we are not inadvertently hitting in the caches
222 // when populating the page tables.
224 InvalidateDataCacheRange (TranslationTable
, EFI_PAGE_SIZE
);
227 ZeroMem (TranslationTable
, EFI_PAGE_SIZE
);
229 if (IsBlockEntry (*Entry
, Level
)) {
231 // We are splitting an existing block entry, so we have to populate
232 // the new table with the attributes of the block entry it replaces.
234 Status
= UpdateRegionMappingRecursive (
235 RegionStart
& ~BlockMask
,
236 (RegionStart
| BlockMask
) + 1,
237 *Entry
& TT_ATTRIBUTES_MASK
,
242 if (EFI_ERROR (Status
)) {
244 // The range we passed to UpdateRegionMappingRecursive () is block
245 // aligned, so it is guaranteed that no further pages were allocated
246 // by it, and so we only have to free the page we allocated here.
248 FreePages (TranslationTable
, 1);
253 TranslationTable
= (VOID
*)(UINTN
)(*Entry
& TT_ADDRESS_MASK_BLOCK_ENTRY
);
257 // Recurse to the next level
259 Status
= UpdateRegionMappingRecursive (
267 if (EFI_ERROR (Status
)) {
268 if (!IsTableEntry (*Entry
, Level
)) {
270 // We are creating a new table entry, so on failure, we can free all
271 // allocations we made recursively, given that the whole subhierarchy
272 // has not been wired into the live page tables yet. (This is not
273 // possible for existing table entries, since we cannot revert the
274 // modifications we made to the subhierarchy it represents.)
276 FreePageTablesRecursive (TranslationTable
, Level
+ 1);
282 if (!IsTableEntry (*Entry
, Level
)) {
283 EntryValue
= (UINTN
)TranslationTable
| TT_TYPE_TABLE_ENTRY
;
288 IsBlockEntry (*Entry
, Level
)
292 EntryValue
= (*Entry
& AttributeClearMask
) | AttributeSetMask
;
293 EntryValue
|= RegionStart
;
294 EntryValue
|= (Level
== 3) ? TT_TYPE_BLOCK_ENTRY_LEVEL3
295 : TT_TYPE_BLOCK_ENTRY
;
297 if (IsTableEntry (*Entry
, Level
)) {
299 // We are replacing a table entry with a block entry. This is only
300 // possible if we are keeping none of the original attributes.
301 // We can free the table entry's page table, and all the ones below
302 // it, since we are dropping the only possible reference to it.
304 ASSERT (AttributeClearMask
== 0);
305 TranslationTable
= (VOID
*)(UINTN
)(*Entry
& TT_ADDRESS_MASK_BLOCK_ENTRY
);
306 ReplaceTableEntry (Entry
, EntryValue
, RegionStart
, TRUE
);
307 FreePageTablesRecursive (TranslationTable
, Level
+ 1);
309 ReplaceTableEntry (Entry
, EntryValue
, RegionStart
, FALSE
);
319 UpdateRegionMapping (
320 IN UINT64 RegionStart
,
321 IN UINT64 RegionLength
,
322 IN UINT64 AttributeSetMask
,
323 IN UINT64 AttributeClearMask
328 if (((RegionStart
| RegionLength
) & EFI_PAGE_MASK
) != 0) {
329 return EFI_INVALID_PARAMETER
;
332 T0SZ
= ArmGetTCR () & TCR_T0SZ_MASK
;
334 return UpdateRegionMappingRecursive (
336 RegionStart
+ RegionLength
,
339 ArmGetTTBR0BaseAddress (),
340 GetRootTableLevel (T0SZ
)
346 FillTranslationTable (
347 IN UINT64
*RootTable
,
348 IN ARM_MEMORY_REGION_DESCRIPTOR
*MemoryRegion
351 return UpdateRegionMapping (
352 MemoryRegion
->VirtualBase
,
353 MemoryRegion
->Length
,
354 ArmMemoryAttributeToPageAttribute (MemoryRegion
->Attributes
) | TT_AF
,
361 GcdAttributeToPageAttribute (
362 IN UINT64 GcdAttributes
365 UINT64 PageAttributes
;
367 switch (GcdAttributes
& EFI_MEMORY_CACHETYPE_MASK
) {
369 PageAttributes
= TT_ATTR_INDX_DEVICE_MEMORY
;
372 PageAttributes
= TT_ATTR_INDX_MEMORY_NON_CACHEABLE
;
375 PageAttributes
= TT_ATTR_INDX_MEMORY_WRITE_THROUGH
| TT_SH_INNER_SHAREABLE
;
378 PageAttributes
= TT_ATTR_INDX_MEMORY_WRITE_BACK
| TT_SH_INNER_SHAREABLE
;
381 PageAttributes
= TT_ATTR_INDX_MASK
;
385 if (((GcdAttributes
& EFI_MEMORY_XP
) != 0) ||
386 ((GcdAttributes
& EFI_MEMORY_CACHETYPE_MASK
) == EFI_MEMORY_UC
))
388 if (ArmReadCurrentEL () == AARCH64_EL2
) {
389 PageAttributes
|= TT_XN_MASK
;
391 PageAttributes
|= TT_UXN_MASK
| TT_PXN_MASK
;
395 if ((GcdAttributes
& EFI_MEMORY_RO
) != 0) {
396 PageAttributes
|= TT_AP_RO_RO
;
399 return PageAttributes
| TT_AF
;
403 ArmSetMemoryAttributes (
404 IN EFI_PHYSICAL_ADDRESS BaseAddress
,
409 UINT64 PageAttributes
;
410 UINT64 PageAttributeMask
;
412 PageAttributes
= GcdAttributeToPageAttribute (Attributes
);
413 PageAttributeMask
= 0;
415 if ((Attributes
& EFI_MEMORY_CACHETYPE_MASK
) == 0) {
417 // No memory type was set in Attributes, so we are going to update the
420 PageAttributes
&= TT_AP_MASK
| TT_UXN_MASK
| TT_PXN_MASK
;
421 PageAttributeMask
= ~(TT_ADDRESS_MASK_BLOCK_ENTRY
| TT_AP_MASK
|
422 TT_PXN_MASK
| TT_XN_MASK
);
425 return UpdateRegionMapping (
435 SetMemoryRegionAttribute (
436 IN EFI_PHYSICAL_ADDRESS BaseAddress
,
438 IN UINT64 Attributes
,
439 IN UINT64 BlockEntryMask
442 return UpdateRegionMapping (BaseAddress
, Length
, Attributes
, BlockEntryMask
);
446 ArmSetMemoryRegionNoExec (
447 IN EFI_PHYSICAL_ADDRESS BaseAddress
,
453 if (ArmReadCurrentEL () == AARCH64_EL1
) {
454 Val
= TT_PXN_MASK
| TT_UXN_MASK
;
459 return SetMemoryRegionAttribute (
463 ~TT_ADDRESS_MASK_BLOCK_ENTRY
468 ArmClearMemoryRegionNoExec (
469 IN EFI_PHYSICAL_ADDRESS BaseAddress
,
475 // XN maps to UXN in the EL1&0 translation regime
476 Mask
= ~(TT_ADDRESS_MASK_BLOCK_ENTRY
| TT_PXN_MASK
| TT_XN_MASK
);
478 return SetMemoryRegionAttribute (
487 ArmSetMemoryRegionReadOnly (
488 IN EFI_PHYSICAL_ADDRESS BaseAddress
,
492 return SetMemoryRegionAttribute (
496 ~TT_ADDRESS_MASK_BLOCK_ENTRY
501 ArmClearMemoryRegionReadOnly (
502 IN EFI_PHYSICAL_ADDRESS BaseAddress
,
506 return SetMemoryRegionAttribute (
510 ~(TT_ADDRESS_MASK_BLOCK_ENTRY
| TT_AP_MASK
)
517 IN ARM_MEMORY_REGION_DESCRIPTOR
*MemoryTable
,
518 OUT VOID
**TranslationTableBase OPTIONAL
,
519 OUT UINTN
*TranslationTableSize OPTIONAL
522 VOID
*TranslationTable
;
523 UINTN MaxAddressBits
;
526 UINTN RootTableEntryCount
;
530 if (MemoryTable
== NULL
) {
531 ASSERT (MemoryTable
!= NULL
);
532 return EFI_INVALID_PARAMETER
;
536 // Limit the virtual address space to what we can actually use: UEFI
537 // mandates a 1:1 mapping, so no point in making the virtual address
538 // space larger than the physical address space. We also have to take
539 // into account the architectural limitations that result from UEFI's
540 // use of 4 KB pages.
542 MaxAddressBits
= MIN (ArmGetPhysicalAddressBits (), MAX_VA_BITS
);
543 MaxAddress
= LShiftU64 (1ULL, MaxAddressBits
) - 1;
545 T0SZ
= 64 - MaxAddressBits
;
546 RootTableEntryCount
= GetRootTableEntryCount (T0SZ
);
549 // Set TCR that allows us to retrieve T0SZ in the subsequent functions
551 // Ideally we will be running at EL2, but should support EL1 as well.
552 // UEFI should not run at EL3.
553 if (ArmReadCurrentEL () == AARCH64_EL2
) {
554 // Note: Bits 23 and 31 are reserved(RES1) bits in TCR_EL2
555 TCR
= T0SZ
| (1UL << 31) | (1UL << 23) | TCR_TG0_4KB
;
557 // Set the Physical Address Size using MaxAddress
558 if (MaxAddress
< SIZE_4GB
) {
560 } else if (MaxAddress
< SIZE_64GB
) {
562 } else if (MaxAddress
< SIZE_1TB
) {
564 } else if (MaxAddress
< SIZE_4TB
) {
566 } else if (MaxAddress
< SIZE_16TB
) {
568 } else if (MaxAddress
< SIZE_256TB
) {
573 "ArmConfigureMmu: The MaxAddress 0x%lX is not supported by this MMU configuration.\n",
576 ASSERT (0); // Bigger than 48-bit memory space are not supported
577 return EFI_UNSUPPORTED
;
579 } else if (ArmReadCurrentEL () == AARCH64_EL1
) {
580 // Due to Cortex-A57 erratum #822227 we must set TG1[1] == 1, regardless of EPD1.
581 TCR
= T0SZ
| TCR_TG0_4KB
| TCR_TG1_4KB
| TCR_EPD1
;
583 // Set the Physical Address Size using MaxAddress
584 if (MaxAddress
< SIZE_4GB
) {
586 } else if (MaxAddress
< SIZE_64GB
) {
588 } else if (MaxAddress
< SIZE_1TB
) {
590 } else if (MaxAddress
< SIZE_4TB
) {
592 } else if (MaxAddress
< SIZE_16TB
) {
594 } else if (MaxAddress
< SIZE_256TB
) {
595 TCR
|= TCR_IPS_256TB
;
599 "ArmConfigureMmu: The MaxAddress 0x%lX is not supported by this MMU configuration.\n",
602 ASSERT (0); // Bigger than 48-bit memory space are not supported
603 return EFI_UNSUPPORTED
;
606 ASSERT (0); // UEFI is only expected to run at EL2 and EL1, not EL3.
607 return EFI_UNSUPPORTED
;
611 // Translation table walks are always cache coherent on ARMv8-A, so cache
612 // maintenance on page tables is never needed. Since there is a risk of
613 // loss of coherency when using mismatched attributes, and given that memory
614 // is mapped cacheable except for extraordinary cases (such as non-coherent
615 // DMA), have the page table walker perform cached accesses as well, and
616 // assert below that that matches the attributes we use for CPU accesses to
619 TCR
|= TCR_SH_INNER_SHAREABLE
|
620 TCR_RGN_OUTER_WRITE_BACK_ALLOC
|
621 TCR_RGN_INNER_WRITE_BACK_ALLOC
;
626 // Allocate pages for translation table
627 TranslationTable
= AllocatePages (1);
628 if (TranslationTable
== NULL
) {
629 return EFI_OUT_OF_RESOURCES
;
633 // We set TTBR0 just after allocating the table to retrieve its location from
634 // the subsequent functions without needing to pass this value across the
635 // functions. The MMU is only enabled after the translation tables are
638 ArmSetTTBR0 (TranslationTable
);
640 if (TranslationTableBase
!= NULL
) {
641 *TranslationTableBase
= TranslationTable
;
644 if (TranslationTableSize
!= NULL
) {
645 *TranslationTableSize
= RootTableEntryCount
* sizeof (UINT64
);
649 // Make sure we are not inadvertently hitting in the caches
650 // when populating the page tables.
652 InvalidateDataCacheRange (
654 RootTableEntryCount
* sizeof (UINT64
)
656 ZeroMem (TranslationTable
, RootTableEntryCount
* sizeof (UINT64
));
658 while (MemoryTable
->Length
!= 0) {
659 Status
= FillTranslationTable (TranslationTable
, MemoryTable
);
660 if (EFI_ERROR (Status
)) {
661 goto FreeTranslationTable
;
668 // EFI_MEMORY_UC ==> MAIR_ATTR_DEVICE_MEMORY
669 // EFI_MEMORY_WC ==> MAIR_ATTR_NORMAL_MEMORY_NON_CACHEABLE
670 // EFI_MEMORY_WT ==> MAIR_ATTR_NORMAL_MEMORY_WRITE_THROUGH
671 // EFI_MEMORY_WB ==> MAIR_ATTR_NORMAL_MEMORY_WRITE_BACK
674 MAIR_ATTR (TT_ATTR_INDX_DEVICE_MEMORY
, MAIR_ATTR_DEVICE_MEMORY
) |
675 MAIR_ATTR (TT_ATTR_INDX_MEMORY_NON_CACHEABLE
, MAIR_ATTR_NORMAL_MEMORY_NON_CACHEABLE
) |
676 MAIR_ATTR (TT_ATTR_INDX_MEMORY_WRITE_THROUGH
, MAIR_ATTR_NORMAL_MEMORY_WRITE_THROUGH
) |
677 MAIR_ATTR (TT_ATTR_INDX_MEMORY_WRITE_BACK
, MAIR_ATTR_NORMAL_MEMORY_WRITE_BACK
)
680 ArmDisableAlignmentCheck ();
681 ArmEnableStackAlignmentCheck ();
682 ArmEnableInstructionCache ();
683 ArmEnableDataCache ();
688 FreeTranslationTable
:
689 FreePages (TranslationTable
, 1);
695 ArmMmuBaseLibConstructor (
699 extern UINT32 ArmReplaceLiveTranslationEntrySize
;
702 // The ArmReplaceLiveTranslationEntry () helper function may be invoked
703 // with the MMU off so we have to ensure that it gets cleaned to the PoC
705 WriteBackDataCacheRange (
706 (VOID
*)(UINTN
)ArmReplaceLiveTranslationEntry
,
707 ArmReplaceLiveTranslationEntrySize
710 return RETURN_SUCCESS
;