[mirror_edk2.git] / ArmPkg / Library / ArmMmuLib / Arm / ArmMmuLibCore.c

/** @file\r
*  File managing the MMU for ARMv7 architecture\r
*\r
*  Copyright (c) 2011-2016, ARM Limited. All rights reserved.\r
*\r
*  SPDX-License-Identifier: BSD-2-Clause-Patent\r
*\r
**/\r
\r
#include <Uefi.h>\r
#include <Chipset/ArmV7.h>\r
#include <Library/BaseMemoryLib.h>\r
#include <Library/CacheMaintenanceLib.h>\r
#include <Library/MemoryAllocationLib.h>\r
#include <Library/ArmLib.h>\r
#include <Library/BaseLib.h>\r
#include <Library/DebugLib.h>\r
#include <Library/PcdLib.h>\r
\r
#define ID_MMFR0_SHARELVL_SHIFT       12\r
#define ID_MMFR0_SHARELVL_MASK       0xf\r
#define ID_MMFR0_SHARELVL_ONE          0\r
#define ID_MMFR0_SHARELVL_TWO          1\r
\r
#define ID_MMFR0_INNERSHR_SHIFT       28\r
#define ID_MMFR0_INNERSHR_MASK       0xf\r
#define ID_MMFR0_OUTERSHR_SHIFT        8\r
#define ID_MMFR0_OUTERSHR_MASK       0xf\r
\r
#define ID_MMFR0_SHR_IMP_UNCACHED      0\r
#define ID_MMFR0_SHR_IMP_HW_COHERENT   1\r
#define ID_MMFR0_SHR_IGNORED         0xf\r
\r
UINTN\r
EFIAPI\r
ArmReadIdMmfr0 (\r
  VOID\r
  );\r
\r
BOOLEAN\r
EFIAPI\r
ArmHasMpExtensions (\r
  VOID\r
  );\r
\r
STATIC\r
BOOLEAN\r
PreferNonshareableMemory (\r
  VOID\r
  )\r
{\r
  UINTN   Mmfr;\r
  UINTN   Val;\r
\r
  if (FeaturePcdGet (PcdNormalMemoryNonshareableOverride)) {\r
    return TRUE;\r
  }\r
\r
  //\r
  // Check whether the innermost level of shareability (the level we will use\r
  // by default to map normal memory) is implemented with hardware coherency\r
  // support. Otherwise, revert to mapping as non-shareable.\r
  //\r
  Mmfr = ArmReadIdMmfr0 ();\r
  switch ((Mmfr >> ID_MMFR0_SHARELVL_SHIFT) & ID_MMFR0_SHARELVL_MASK) {\r
  case ID_MMFR0_SHARELVL_ONE:\r
    // one level of shareability\r
    Val = (Mmfr >> ID_MMFR0_OUTERSHR_SHIFT) & ID_MMFR0_OUTERSHR_MASK;\r
    break;\r
  case ID_MMFR0_SHARELVL_TWO:\r
    // two levels of shareability\r
    Val = (Mmfr >> ID_MMFR0_INNERSHR_SHIFT) & ID_MMFR0_INNERSHR_MASK;\r
    break;\r
  default:\r
    // unexpected value -> shareable is the safe option\r
    ASSERT (FALSE);\r
    return FALSE;\r
  }\r
  return Val != ID_MMFR0_SHR_IMP_HW_COHERENT;\r
}\r
\r
STATIC\r
VOID\r
PopulateLevel2PageTable (\r
  IN UINT32                         *SectionEntry,\r
  IN UINT32                         PhysicalBase,\r
  IN UINT32                         RemainLength,\r
  IN ARM_MEMORY_REGION_ATTRIBUTES   Attributes\r
  )\r
{\r
  UINT32* PageEntry;\r
  UINT32  Pages;\r
  UINT32  Index;\r
  UINT32  PageAttributes;\r
  UINT32  SectionDescriptor;\r
  UINT32  TranslationTable;\r
  UINT32  BaseSectionAddress;\r
  UINT32  FirstPageOffset;\r
\r
  switch (Attributes) {\r
    case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK:\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK:\r
      PageAttributes = TT_DESCRIPTOR_PAGE_WRITE_BACK;\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK_NONSHAREABLE:\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK_NONSHAREABLE:\r
      PageAttributes = TT_DESCRIPTOR_PAGE_WRITE_BACK;\r
      PageAttributes &= ~TT_DESCRIPTOR_PAGE_S_SHARED;\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_THROUGH:\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_THROUGH:\r
      PageAttributes = TT_DESCRIPTOR_PAGE_WRITE_THROUGH;\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_DEVICE:\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_DEVICE:\r
      PageAttributes = TT_DESCRIPTOR_PAGE_DEVICE;\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_UNCACHED_UNBUFFERED:\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_UNCACHED_UNBUFFERED:\r
      PageAttributes = TT_DESCRIPTOR_PAGE_UNCACHED;\r
      break;\r
    default:\r
      PageAttributes = TT_DESCRIPTOR_PAGE_UNCACHED;\r
      break;\r
  }\r
\r
  if (PreferNonshareableMemory ()) {\r
    PageAttributes &= ~TT_DESCRIPTOR_PAGE_S_SHARED;\r
  }\r
\r
  // Check if the Section Entry has already been populated. Otherwise attach a\r
  // Level 2 Translation Table to it\r
  if (*SectionEntry != 0) {\r
    // The entry must be a page table. Otherwise it exists an overlapping in the memory map\r
    if (TT_DESCRIPTOR_SECTION_TYPE_IS_PAGE_TABLE(*SectionEntry)) {\r
      TranslationTable = *SectionEntry & TT_DESCRIPTOR_SECTION_PAGETABLE_ADDRESS_MASK;\r
    } else if ((*SectionEntry & TT_DESCRIPTOR_SECTION_TYPE_MASK) == TT_DESCRIPTOR_SECTION_TYPE_SECTION) {\r
      // Case where a virtual memory map descriptor overlapped a section entry\r
\r
      // Allocate a Level2 Page Table for this Section\r
      TranslationTable = (UINTN)AllocateAlignedPages (\r
                                  EFI_SIZE_TO_PAGES (TRANSLATION_TABLE_PAGE_SIZE),\r
                                  TRANSLATION_TABLE_PAGE_ALIGNMENT);\r
\r
      // Translate the Section Descriptor into Page Descriptor\r
      SectionDescriptor = TT_DESCRIPTOR_PAGE_TYPE_PAGE | ConvertSectionAttributesToPageAttributes (*SectionEntry, FALSE);\r
\r
      BaseSectionAddress = TT_DESCRIPTOR_SECTION_BASE_ADDRESS(*SectionEntry);\r
\r
      //\r
      // Make sure we are not inadvertently hitting in the caches\r
      // when populating the page tables\r
      //\r
      InvalidateDataCacheRange ((VOID *)TranslationTable,\r
        TRANSLATION_TABLE_PAGE_SIZE);\r
\r
      // Populate the new Level2 Page Table for the section\r
      PageEntry = (UINT32*)TranslationTable;\r
      for (Index = 0; Index < TRANSLATION_TABLE_PAGE_COUNT; Index++) {\r
        PageEntry[Index] = TT_DESCRIPTOR_PAGE_BASE_ADDRESS(BaseSectionAddress + (Index << 12)) | SectionDescriptor;\r
      }\r
\r
      // Overwrite the section entry to point to the new Level2 Translation Table\r
      *SectionEntry = (TranslationTable & TT_DESCRIPTOR_SECTION_PAGETABLE_ADDRESS_MASK) |\r
          (IS_ARM_MEMORY_REGION_ATTRIBUTES_SECURE(Attributes) ? (1 << 3) : 0) |\r
          TT_DESCRIPTOR_SECTION_TYPE_PAGE_TABLE;\r
    } else {\r
      // We do not support the other section type (16MB Section)\r
      ASSERT(0);\r
      return;\r
    }\r
  } else {\r
    TranslationTable = (UINTN)AllocateAlignedPages (\r
                                EFI_SIZE_TO_PAGES (TRANSLATION_TABLE_PAGE_SIZE),\r
                                TRANSLATION_TABLE_PAGE_ALIGNMENT);\r
    //\r
    // Make sure we are not inadvertently hitting in the caches\r
    // when populating the page tables\r
    //\r
    InvalidateDataCacheRange ((VOID *)TranslationTable,\r
      TRANSLATION_TABLE_PAGE_SIZE);\r
    ZeroMem ((VOID *)TranslationTable, TRANSLATION_TABLE_PAGE_SIZE);\r
\r
    *SectionEntry = (TranslationTable & TT_DESCRIPTOR_SECTION_PAGETABLE_ADDRESS_MASK) |\r
        (IS_ARM_MEMORY_REGION_ATTRIBUTES_SECURE(Attributes) ? (1 << 3) : 0) |\r
        TT_DESCRIPTOR_SECTION_TYPE_PAGE_TABLE;\r
  }\r
\r
  FirstPageOffset = (PhysicalBase & TT_DESCRIPTOR_PAGE_INDEX_MASK) >> TT_DESCRIPTOR_PAGE_BASE_SHIFT;\r
  PageEntry = (UINT32 *)TranslationTable + FirstPageOffset;\r
  Pages     = RemainLength / TT_DESCRIPTOR_PAGE_SIZE;\r
\r
  ASSERT (FirstPageOffset + Pages <= TRANSLATION_TABLE_PAGE_COUNT);\r
\r
  for (Index = 0; Index < Pages; Index++) {\r
    *PageEntry++     =  TT_DESCRIPTOR_PAGE_BASE_ADDRESS(PhysicalBase) | PageAttributes;\r
    PhysicalBase += TT_DESCRIPTOR_PAGE_SIZE;\r
  }\r
\r
  //\r
  // Invalidate again to ensure that any line fetches that may have occurred\r
  // [speculatively] since the previous invalidate are evicted again.\r
  //\r
  ArmDataMemoryBarrier ();\r
  InvalidateDataCacheRange ((UINT32 *)TranslationTable + FirstPageOffset,\r
    RemainLength / TT_DESCRIPTOR_PAGE_SIZE * sizeof (*PageEntry));\r
}\r
\r
STATIC\r
VOID\r
FillTranslationTable (\r
  IN  UINT32                        *TranslationTable,\r
  IN  ARM_MEMORY_REGION_DESCRIPTOR  *MemoryRegion\r
  )\r
{\r
  UINT32  *SectionEntry;\r
  UINT32  Attributes;\r
  UINT32  PhysicalBase;\r
  UINT64  RemainLength;\r
  UINT32  PageMapLength;\r
\r
  ASSERT(MemoryRegion->Length > 0);\r
\r
  if (MemoryRegion->PhysicalBase >= SIZE_4GB) {\r
    return;\r
  }\r
\r
  PhysicalBase = (UINT32)MemoryRegion->PhysicalBase;\r
  RemainLength = MIN(MemoryRegion->Length, SIZE_4GB - PhysicalBase);\r
\r
  switch (MemoryRegion->Attributes) {\r
    case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK:\r
      Attributes = TT_DESCRIPTOR_SECTION_WRITE_BACK(0);\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK_NONSHAREABLE:\r
      Attributes = TT_DESCRIPTOR_SECTION_WRITE_BACK(0);\r
      Attributes &= ~TT_DESCRIPTOR_SECTION_S_SHARED;\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_THROUGH:\r
      Attributes = TT_DESCRIPTOR_SECTION_WRITE_THROUGH(0);\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_DEVICE:\r
      Attributes = TT_DESCRIPTOR_SECTION_DEVICE(0);\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_UNCACHED_UNBUFFERED:\r
      Attributes = TT_DESCRIPTOR_SECTION_UNCACHED(0);\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK:\r
      Attributes = TT_DESCRIPTOR_SECTION_WRITE_BACK(1);\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK_NONSHAREABLE:\r
      Attributes = TT_DESCRIPTOR_SECTION_WRITE_BACK(1);\r
      Attributes &= ~TT_DESCRIPTOR_SECTION_S_SHARED;\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_THROUGH:\r
      Attributes = TT_DESCRIPTOR_SECTION_WRITE_THROUGH(1);\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_DEVICE:\r
      Attributes = TT_DESCRIPTOR_SECTION_DEVICE(1);\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_UNCACHED_UNBUFFERED:\r
      Attributes = TT_DESCRIPTOR_SECTION_UNCACHED(1);\r
      break;\r
    default:\r
      Attributes = TT_DESCRIPTOR_SECTION_UNCACHED(0);\r
      break;\r
  }\r
\r
  if (PreferNonshareableMemory ()) {\r
    Attributes &= ~TT_DESCRIPTOR_SECTION_S_SHARED;\r
  }\r
\r
  // Get the first section entry for this mapping\r
  SectionEntry    = TRANSLATION_TABLE_ENTRY_FOR_VIRTUAL_ADDRESS(TranslationTable, MemoryRegion->VirtualBase);\r
\r
  while (RemainLength != 0) {\r
    if (PhysicalBase % TT_DESCRIPTOR_SECTION_SIZE == 0 &&\r
        RemainLength >= TT_DESCRIPTOR_SECTION_SIZE) {\r
      // Case: Physical address aligned on the Section Size (1MB) && the length\r
      // is greater than the Section Size\r
      *SectionEntry = TT_DESCRIPTOR_SECTION_BASE_ADDRESS(PhysicalBase) | Attributes;\r
\r
      //\r
      // Issue a DMB to ensure that the page table entry update made it to\r
      // memory before we issue the invalidate, otherwise, a subsequent\r
      // speculative fetch could observe the old value.\r
      //\r
      ArmDataMemoryBarrier ();\r
      ArmInvalidateDataCacheEntryByMVA ((UINTN)SectionEntry++);\r
\r
      PhysicalBase += TT_DESCRIPTOR_SECTION_SIZE;\r
      RemainLength -= TT_DESCRIPTOR_SECTION_SIZE;\r
    } else {\r
      PageMapLength = MIN ((UINT32)RemainLength, TT_DESCRIPTOR_SECTION_SIZE -\r
                                         (PhysicalBase % TT_DESCRIPTOR_SECTION_SIZE));\r
\r
      // Case: Physical address aligned on the Section Size (1MB) && the length\r
      //       does not fill a section\r
      // Case: Physical address NOT aligned on the Section Size (1MB)\r
      PopulateLevel2PageTable (SectionEntry, PhysicalBase, PageMapLength,\r
        MemoryRegion->Attributes);\r
\r
      //\r
      // Issue a DMB to ensure that the page table entry update made it to\r
      // memory before we issue the invalidate, otherwise, a subsequent\r
      // speculative fetch could observe the old value.\r
      //\r
      ArmDataMemoryBarrier ();\r
      ArmInvalidateDataCacheEntryByMVA ((UINTN)SectionEntry++);\r
\r
      // If it is the last entry\r
      if (RemainLength < TT_DESCRIPTOR_SECTION_SIZE) {\r
        break;\r
      }\r
\r
      PhysicalBase += PageMapLength;\r
      RemainLength -= PageMapLength;\r
    }\r
  }\r
}\r
\r
RETURN_STATUS\r
EFIAPI\r
ArmConfigureMmu (\r
  IN  ARM_MEMORY_REGION_DESCRIPTOR  *MemoryTable,\r
  OUT VOID                         **TranslationTableBase OPTIONAL,\r
  OUT UINTN                         *TranslationTableSize OPTIONAL\r
  )\r
{\r
  VOID                          *TranslationTable;\r
  UINT32                        TTBRAttributes;\r
\r
  TranslationTable = AllocateAlignedPages (\r
                       EFI_SIZE_TO_PAGES (TRANSLATION_TABLE_SECTION_SIZE),\r
                       TRANSLATION_TABLE_SECTION_ALIGNMENT);\r
  if (TranslationTable == NULL) {\r
    return RETURN_OUT_OF_RESOURCES;\r
  }\r
\r
  if (TranslationTableBase != NULL) {\r
    *TranslationTableBase = TranslationTable;\r
  }\r
\r
  if (TranslationTableSize != NULL) {\r
    *TranslationTableSize = TRANSLATION_TABLE_SECTION_SIZE;\r
  }\r
\r
  //\r
  // Make sure we are not inadvertently hitting in the caches\r
  // when populating the page tables\r
  //\r
  InvalidateDataCacheRange (TranslationTable, TRANSLATION_TABLE_SECTION_SIZE);\r
  ZeroMem (TranslationTable, TRANSLATION_TABLE_SECTION_SIZE);\r
\r
  while (MemoryTable->Length != 0) {\r
    FillTranslationTable (TranslationTable, MemoryTable);\r
    MemoryTable++;\r
  }\r
\r
  TTBRAttributes = ArmHasMpExtensions () ? TTBR_MP_WRITE_BACK_ALLOC\r
                                         : TTBR_WRITE_BACK_ALLOC;\r
  if (TTBRAttributes & TTBR_SHAREABLE) {\r
    if (PreferNonshareableMemory ()) {\r
      TTBRAttributes ^= TTBR_SHAREABLE;\r
    } else {\r
      //\r
      // Unlike the S bit in the short descriptors, which implies inner shareable\r
      // on an implementation that supports two levels, the meaning of the S bit\r
      // in the TTBR depends on the NOS bit, which defaults to Outer Shareable.\r
      // However, we should only set this bit after we have confirmed that the\r
      // implementation supports multiple levels, or else the NOS bit is UNK/SBZP\r
      //\r
      if (((ArmReadIdMmfr0 () >> 12) & 0xf) != 0) {\r
        TTBRAttributes |= TTBR_NOT_OUTER_SHAREABLE;\r
      }\r
    }\r
  }\r
\r
  ArmSetTTBR0 ((VOID *)((UINTN)TranslationTable | TTBRAttributes));\r
\r
  //\r
  // The TTBCR register value is undefined at reset in the Non-Secure world.\r
  // Writing 0 has the effect of:\r
  //   Clearing EAE: Use short descriptors, as mandated by specification.\r
  //   Clearing PD0 and PD1: Translation Table Walk Disable is off.\r
  //   Clearing N: Perform all translation table walks through TTBR0.\r
  //               (0 is the default reset value in systems not implementing\r
  //               the Security Extensions.)\r
  //\r
  ArmSetTTBCR (0);\r
\r
  ArmSetDomainAccessControl (DOMAIN_ACCESS_CONTROL_NONE(15) |\r
                             DOMAIN_ACCESS_CONTROL_NONE(14) |\r
                             DOMAIN_ACCESS_CONTROL_NONE(13) |\r
                             DOMAIN_ACCESS_CONTROL_NONE(12) |\r
                             DOMAIN_ACCESS_CONTROL_NONE(11) |\r
                             DOMAIN_ACCESS_CONTROL_NONE(10) |\r
                             DOMAIN_ACCESS_CONTROL_NONE( 9) |\r
                             DOMAIN_ACCESS_CONTROL_NONE( 8) |\r
                             DOMAIN_ACCESS_CONTROL_NONE( 7) |\r
                             DOMAIN_ACCESS_CONTROL_NONE( 6) |\r
                             DOMAIN_ACCESS_CONTROL_NONE( 5) |\r
                             DOMAIN_ACCESS_CONTROL_NONE( 4) |\r
                             DOMAIN_ACCESS_CONTROL_NONE( 3) |\r
                             DOMAIN_ACCESS_CONTROL_NONE( 2) |\r
                             DOMAIN_ACCESS_CONTROL_NONE( 1) |\r
                             DOMAIN_ACCESS_CONTROL_CLIENT(0));\r
\r
  ArmEnableInstructionCache();\r
  ArmEnableDataCache();\r
  ArmEnableMmu();\r
  return RETURN_SUCCESS;\r
}\r
Commit	Line	Data
d7f03464 AB	1	/** @file\r
	2	* File managing the MMU for ARMv7 architecture\r
	3	*\r
	4	* Copyright (c) 2011-2016, ARM Limited. All rights reserved.\r
	5	*\r
4059386c	6	* SPDX-License-Identifier: BSD-2-Clause-Patent\r
d7f03464 AB	7	*\r
	8	**/\r
	9	\r
	10	#include <Uefi.h>\r
	11	#include <Chipset/ArmV7.h>\r
	12	#include <Library/BaseMemoryLib.h>\r
521f3ced	13	#include <Library/CacheMaintenanceLib.h>\r
d7f03464 AB	14	#include <Library/MemoryAllocationLib.h>\r
	15	#include <Library/ArmLib.h>\r
	16	#include <Library/BaseLib.h>\r
	17	#include <Library/DebugLib.h>\r
	18	#include <Library/PcdLib.h>\r
	19	\r
	20	#define ID_MMFR0_SHARELVL_SHIFT 12\r
	21	#define ID_MMFR0_SHARELVL_MASK 0xf\r
	22	#define ID_MMFR0_SHARELVL_ONE 0\r
	23	#define ID_MMFR0_SHARELVL_TWO 1\r
	24	\r
	25	#define ID_MMFR0_INNERSHR_SHIFT 28\r
	26	#define ID_MMFR0_INNERSHR_MASK 0xf\r
	27	#define ID_MMFR0_OUTERSHR_SHIFT 8\r
	28	#define ID_MMFR0_OUTERSHR_MASK 0xf\r
	29	\r
	30	#define ID_MMFR0_SHR_IMP_UNCACHED 0\r
	31	#define ID_MMFR0_SHR_IMP_HW_COHERENT 1\r
	32	#define ID_MMFR0_SHR_IGNORED 0xf\r
	33	\r
	34	UINTN\r
	35	EFIAPI\r
	36	ArmReadIdMmfr0 (\r
	37	VOID\r
	38	);\r
	39	\r
	40	BOOLEAN\r
	41	EFIAPI\r
	42	ArmHasMpExtensions (\r
	43	VOID\r
	44	);\r
	45	\r
d7f03464 AB	46	STATIC\r
	47	BOOLEAN\r
	48	PreferNonshareableMemory (\r
	49	VOID\r
	50	)\r
	51	{\r
	52	UINTN Mmfr;\r
	53	UINTN Val;\r
	54	\r
	55	if (FeaturePcdGet (PcdNormalMemoryNonshareableOverride)) {\r
	56	return TRUE;\r
	57	}\r
	58	\r
	59	//\r
	60	// Check whether the innermost level of shareability (the level we will use\r
	61	// by default to map normal memory) is implemented with hardware coherency\r
	62	// support. Otherwise, revert to mapping as non-shareable.\r
	63	//\r
	64	Mmfr = ArmReadIdMmfr0 ();\r
	65	switch ((Mmfr >> ID_MMFR0_SHARELVL_SHIFT) & ID_MMFR0_SHARELVL_MASK) {\r
	66	case ID_MMFR0_SHARELVL_ONE:\r
	67	// one level of shareability\r
	68	Val = (Mmfr >> ID_MMFR0_OUTERSHR_SHIFT) & ID_MMFR0_OUTERSHR_MASK;\r
	69	break;\r
	70	case ID_MMFR0_SHARELVL_TWO:\r
	71	// two levels of shareability\r
	72	Val = (Mmfr >> ID_MMFR0_INNERSHR_SHIFT) & ID_MMFR0_INNERSHR_MASK;\r
	73	break;\r
	74	default:\r
	75	// unexpected value -> shareable is the safe option\r
	76	ASSERT (FALSE);\r
	77	return FALSE;\r
	78	}\r
	79	return Val != ID_MMFR0_SHR_IMP_HW_COHERENT;\r
	80	}\r
	81	\r
	82	STATIC\r
	83	VOID\r
	84	PopulateLevel2PageTable (\r
	85	IN UINT32 *SectionEntry,\r
	86	IN UINT32 PhysicalBase,\r
	87	IN UINT32 RemainLength,\r
	88	IN ARM_MEMORY_REGION_ATTRIBUTES Attributes\r
	89	)\r
	90	{\r
	91	UINT32* PageEntry;\r
	92	UINT32 Pages;\r
	93	UINT32 Index;\r
	94	UINT32 PageAttributes;\r
	95	UINT32 SectionDescriptor;\r
	96	UINT32 TranslationTable;\r
	97	UINT32 BaseSectionAddress;\r
889c7ca1	98	UINT32 FirstPageOffset;\r
d7f03464 AB	99	\r
	100	switch (Attributes) {\r
	101	case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK:\r
	102	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK:\r
	103	PageAttributes = TT_DESCRIPTOR_PAGE_WRITE_BACK;\r
	104	break;\r
e3ad54fa AB	105	case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK_NONSHAREABLE:\r
	106	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK_NONSHAREABLE:\r
	107	PageAttributes = TT_DESCRIPTOR_PAGE_WRITE_BACK;\r
	108	PageAttributes &= ~TT_DESCRIPTOR_PAGE_S_SHARED;\r
	109	break;\r
d7f03464 AB	110	case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_THROUGH:\r
	111	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_THROUGH:\r
	112	PageAttributes = TT_DESCRIPTOR_PAGE_WRITE_THROUGH;\r
	113	break;\r
	114	case ARM_MEMORY_REGION_ATTRIBUTE_DEVICE:\r
	115	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_DEVICE:\r
	116	PageAttributes = TT_DESCRIPTOR_PAGE_DEVICE;\r
	117	break;\r
	118	case ARM_MEMORY_REGION_ATTRIBUTE_UNCACHED_UNBUFFERED:\r
	119	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_UNCACHED_UNBUFFERED:\r
	120	PageAttributes = TT_DESCRIPTOR_PAGE_UNCACHED;\r
	121	break;\r
	122	default:\r
	123	PageAttributes = TT_DESCRIPTOR_PAGE_UNCACHED;\r
	124	break;\r
	125	}\r
	126	\r
	127	if (PreferNonshareableMemory ()) {\r
	128	PageAttributes &= ~TT_DESCRIPTOR_PAGE_S_SHARED;\r
	129	}\r
	130	\r
	131	// Check if the Section Entry has already been populated. Otherwise attach a\r
	132	// Level 2 Translation Table to it\r
	133	if (*SectionEntry != 0) {\r
	134	// The entry must be a page table. Otherwise it exists an overlapping in the memory map\r
	135	if (TT_DESCRIPTOR_SECTION_TYPE_IS_PAGE_TABLE(*SectionEntry)) {\r
	136	TranslationTable = *SectionEntry & TT_DESCRIPTOR_SECTION_PAGETABLE_ADDRESS_MASK;\r
	137	} else if ((*SectionEntry & TT_DESCRIPTOR_SECTION_TYPE_MASK) == TT_DESCRIPTOR_SECTION_TYPE_SECTION) {\r
	138	// Case where a virtual memory map descriptor overlapped a section entry\r
	139	\r
	140	// Allocate a Level2 Page Table for this Section\r
825c3e2c AB	141	TranslationTable = (UINTN)AllocateAlignedPages (\r
	142	EFI_SIZE_TO_PAGES (TRANSLATION_TABLE_PAGE_SIZE),\r
	143	TRANSLATION_TABLE_PAGE_ALIGNMENT);\r
d7f03464 AB	144	\r
	145	// Translate the Section Descriptor into Page Descriptor\r
	146	SectionDescriptor = TT_DESCRIPTOR_PAGE_TYPE_PAGE \| ConvertSectionAttributesToPageAttributes (*SectionEntry, FALSE);\r
	147	\r
	148	BaseSectionAddress = TT_DESCRIPTOR_SECTION_BASE_ADDRESS(*SectionEntry);\r
	149	\r
02d7797d AB	150	//\r
	151	// Make sure we are not inadvertently hitting in the caches\r
	152	// when populating the page tables\r
	153	//\r
	154	InvalidateDataCacheRange ((VOID *)TranslationTable,\r
	155	TRANSLATION_TABLE_PAGE_SIZE);\r
	156	\r
d7f03464 AB	157	// Populate the new Level2 Page Table for the section\r
	158	PageEntry = (UINT32*)TranslationTable;\r
	159	for (Index = 0; Index < TRANSLATION_TABLE_PAGE_COUNT; Index++) {\r
	160	PageEntry[Index] = TT_DESCRIPTOR_PAGE_BASE_ADDRESS(BaseSectionAddress + (Index << 12)) \| SectionDescriptor;\r
	161	}\r
	162	\r
	163	// Overwrite the section entry to point to the new Level2 Translation Table\r
	164	*SectionEntry = (TranslationTable & TT_DESCRIPTOR_SECTION_PAGETABLE_ADDRESS_MASK) \|\r
	165	(IS_ARM_MEMORY_REGION_ATTRIBUTES_SECURE(Attributes) ? (1 << 3) : 0) \|\r
	166	TT_DESCRIPTOR_SECTION_TYPE_PAGE_TABLE;\r
	167	} else {\r
	168	// We do not support the other section type (16MB Section)\r
	169	ASSERT(0);\r
	170	return;\r
	171	}\r
	172	} else {\r
825c3e2c AB	173	TranslationTable = (UINTN)AllocateAlignedPages (\r
	174	EFI_SIZE_TO_PAGES (TRANSLATION_TABLE_PAGE_SIZE),\r
	175	TRANSLATION_TABLE_PAGE_ALIGNMENT);\r
02d7797d AB	176	//\r
	177	// Make sure we are not inadvertently hitting in the caches\r
	178	// when populating the page tables\r
	179	//\r
	180	InvalidateDataCacheRange ((VOID *)TranslationTable,\r
	181	TRANSLATION_TABLE_PAGE_SIZE);\r
d7f03464 AB	182	ZeroMem ((VOID *)TranslationTable, TRANSLATION_TABLE_PAGE_SIZE);\r
	183	\r
	184	*SectionEntry = (TranslationTable & TT_DESCRIPTOR_SECTION_PAGETABLE_ADDRESS_MASK) \|\r
	185	(IS_ARM_MEMORY_REGION_ATTRIBUTES_SECURE(Attributes) ? (1 << 3) : 0) \|\r
	186	TT_DESCRIPTOR_SECTION_TYPE_PAGE_TABLE;\r
	187	}\r
	188	\r
889c7ca1 MZ	189	FirstPageOffset = (PhysicalBase & TT_DESCRIPTOR_PAGE_INDEX_MASK) >> TT_DESCRIPTOR_PAGE_BASE_SHIFT;\r
889c7ca1 MZ	190	PageEntry = (UINT32 *)TranslationTable + FirstPageOffset;\r
d7f03464 AB	191	Pages = RemainLength / TT_DESCRIPTOR_PAGE_SIZE;\r
d7f03464 AB	192	\r
889c7ca1 MZ	193	ASSERT (FirstPageOffset + Pages <= TRANSLATION_TABLE_PAGE_COUNT);\r
889c7ca1 MZ	194	\r
d7f03464 AB	195	for (Index = 0; Index < Pages; Index++) {\r
	196	*PageEntry++ = TT_DESCRIPTOR_PAGE_BASE_ADDRESS(PhysicalBase) \| PageAttributes;\r
	197	PhysicalBase += TT_DESCRIPTOR_PAGE_SIZE;\r
	198	}\r
	199	\r
02d7797d AB	200	//\r
	201	// Invalidate again to ensure that any line fetches that may have occurred\r
	202	// [speculatively] since the previous invalidate are evicted again.\r
	203	//\r
	204	ArmDataMemoryBarrier ();\r
	205	InvalidateDataCacheRange ((UINT32 *)TranslationTable + FirstPageOffset,\r
	206	RemainLength / TT_DESCRIPTOR_PAGE_SIZE * sizeof (*PageEntry));\r
d7f03464 AB	207	}\r
	208	\r
	209	STATIC\r
	210	VOID\r
	211	FillTranslationTable (\r
	212	IN UINT32 *TranslationTable,\r
	213	IN ARM_MEMORY_REGION_DESCRIPTOR *MemoryRegion\r
	214	)\r
	215	{\r
	216	UINT32 *SectionEntry;\r
	217	UINT32 Attributes;\r
	218	UINT32 PhysicalBase;\r
	219	UINT64 RemainLength;\r
889c7ca1	220	UINT32 PageMapLength;\r
d7f03464 AB	221	\r
	222	ASSERT(MemoryRegion->Length > 0);\r
	223	\r
	224	if (MemoryRegion->PhysicalBase >= SIZE_4GB) {\r
	225	return;\r
	226	}\r
	227	\r
2bdc9042	228	PhysicalBase = (UINT32)MemoryRegion->PhysicalBase;\r
d7f03464 AB	229	RemainLength = MIN(MemoryRegion->Length, SIZE_4GB - PhysicalBase);\r
	230	\r
	231	switch (MemoryRegion->Attributes) {\r
	232	case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK:\r
	233	Attributes = TT_DESCRIPTOR_SECTION_WRITE_BACK(0);\r
	234	break;\r
e3ad54fa AB	235	case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK_NONSHAREABLE:\r
	236	Attributes = TT_DESCRIPTOR_SECTION_WRITE_BACK(0);\r
	237	Attributes &= ~TT_DESCRIPTOR_SECTION_S_SHARED;\r
	238	break;\r
d7f03464 AB	239	case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_THROUGH:\r
	240	Attributes = TT_DESCRIPTOR_SECTION_WRITE_THROUGH(0);\r
	241	break;\r
	242	case ARM_MEMORY_REGION_ATTRIBUTE_DEVICE:\r
	243	Attributes = TT_DESCRIPTOR_SECTION_DEVICE(0);\r
	244	break;\r
	245	case ARM_MEMORY_REGION_ATTRIBUTE_UNCACHED_UNBUFFERED:\r
	246	Attributes = TT_DESCRIPTOR_SECTION_UNCACHED(0);\r
	247	break;\r
	248	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK:\r
	249	Attributes = TT_DESCRIPTOR_SECTION_WRITE_BACK(1);\r
	250	break;\r
e3ad54fa AB	251	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK_NONSHAREABLE:\r
	252	Attributes = TT_DESCRIPTOR_SECTION_WRITE_BACK(1);\r
	253	Attributes &= ~TT_DESCRIPTOR_SECTION_S_SHARED;\r
	254	break;\r
d7f03464 AB	255	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_THROUGH:\r
	256	Attributes = TT_DESCRIPTOR_SECTION_WRITE_THROUGH(1);\r
	257	break;\r
	258	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_DEVICE:\r
	259	Attributes = TT_DESCRIPTOR_SECTION_DEVICE(1);\r
	260	break;\r
	261	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_UNCACHED_UNBUFFERED:\r
	262	Attributes = TT_DESCRIPTOR_SECTION_UNCACHED(1);\r
	263	break;\r
	264	default:\r
	265	Attributes = TT_DESCRIPTOR_SECTION_UNCACHED(0);\r
	266	break;\r
	267	}\r
	268	\r
	269	if (PreferNonshareableMemory ()) {\r
	270	Attributes &= ~TT_DESCRIPTOR_SECTION_S_SHARED;\r
	271	}\r
	272	\r
	273	// Get the first section entry for this mapping\r
	274	SectionEntry = TRANSLATION_TABLE_ENTRY_FOR_VIRTUAL_ADDRESS(TranslationTable, MemoryRegion->VirtualBase);\r
	275	\r
	276	while (RemainLength != 0) {\r
889c7ca1 MZ	277	if (PhysicalBase % TT_DESCRIPTOR_SECTION_SIZE == 0 &&\r
	278	RemainLength >= TT_DESCRIPTOR_SECTION_SIZE) {\r
	279	// Case: Physical address aligned on the Section Size (1MB) && the length\r
	280	// is greater than the Section Size\r
02d7797d AB	281	*SectionEntry = TT_DESCRIPTOR_SECTION_BASE_ADDRESS(PhysicalBase) \| Attributes;\r
	282	\r
	283	//\r
	284	// Issue a DMB to ensure that the page table entry update made it to\r
	285	// memory before we issue the invalidate, otherwise, a subsequent\r
	286	// speculative fetch could observe the old value.\r
	287	//\r
	288	ArmDataMemoryBarrier ();\r
	289	ArmInvalidateDataCacheEntryByMVA ((UINTN)SectionEntry++);\r
	290	\r
889c7ca1 MZ	291	PhysicalBase += TT_DESCRIPTOR_SECTION_SIZE;\r
889c7ca1 MZ	292	RemainLength -= TT_DESCRIPTOR_SECTION_SIZE;\r
d7f03464	293	} else {\r
2bdc9042	294	PageMapLength = MIN ((UINT32)RemainLength, TT_DESCRIPTOR_SECTION_SIZE -\r
28ce4cb3	295	(PhysicalBase % TT_DESCRIPTOR_SECTION_SIZE));\r
889c7ca1 MZ	296	\r
	297	// Case: Physical address aligned on the Section Size (1MB) && the length\r
	298	// does not fill a section\r
d7f03464	299	// Case: Physical address NOT aligned on the Section Size (1MB)\r
02d7797d	300	PopulateLevel2PageTable (SectionEntry, PhysicalBase, PageMapLength,\r
889c7ca1	301	MemoryRegion->Attributes);\r
d7f03464	302	\r
02d7797d AB	303	//\r
	304	// Issue a DMB to ensure that the page table entry update made it to\r
	305	// memory before we issue the invalidate, otherwise, a subsequent\r
	306	// speculative fetch could observe the old value.\r
	307	//\r
	308	ArmDataMemoryBarrier ();\r
	309	ArmInvalidateDataCacheEntryByMVA ((UINTN)SectionEntry++);\r
	310	\r
d7f03464 AB	311	// If it is the last entry\r
	312	if (RemainLength < TT_DESCRIPTOR_SECTION_SIZE) {\r
	313	break;\r
	314	}\r
889c7ca1 MZ	315	\r
	316	PhysicalBase += PageMapLength;\r
	317	RemainLength -= PageMapLength;\r
d7f03464	318	}\r
d7f03464 AB	319	}\r
	320	}\r
	321	\r
	322	RETURN_STATUS\r
	323	EFIAPI\r
	324	ArmConfigureMmu (\r
	325	IN ARM_MEMORY_REGION_DESCRIPTOR *MemoryTable,\r
	326	OUT VOID **TranslationTableBase OPTIONAL,\r
	327	OUT UINTN *TranslationTableSize OPTIONAL\r
	328	)\r
	329	{\r
825c3e2c	330	VOID *TranslationTable;\r
d7f03464 AB	331	UINT32 TTBRAttributes;\r
d7f03464 AB	332	\r
825c3e2c AB	333	TranslationTable = AllocateAlignedPages (\r
	334	EFI_SIZE_TO_PAGES (TRANSLATION_TABLE_SECTION_SIZE),\r
	335	TRANSLATION_TABLE_SECTION_ALIGNMENT);\r
d7f03464 AB	336	if (TranslationTable == NULL) {\r
	337	return RETURN_OUT_OF_RESOURCES;\r
	338	}\r
d7f03464 AB	339	\r
	340	if (TranslationTableBase != NULL) {\r
	341	*TranslationTableBase = TranslationTable;\r
	342	}\r
	343	\r
	344	if (TranslationTableSize != NULL) {\r
	345	*TranslationTableSize = TRANSLATION_TABLE_SECTION_SIZE;\r
	346	}\r
	347	\r
02d7797d AB	348	//\r
	349	// Make sure we are not inadvertently hitting in the caches\r
	350	// when populating the page tables\r
	351	//\r
	352	InvalidateDataCacheRange (TranslationTable, TRANSLATION_TABLE_SECTION_SIZE);\r
d7f03464 AB	353	ZeroMem (TranslationTable, TRANSLATION_TABLE_SECTION_SIZE);\r
d7f03464 AB	354	\r
d7f03464	355	while (MemoryTable->Length != 0) {\r
d7f03464 AB	356	FillTranslationTable (TranslationTable, MemoryTable);\r
	357	MemoryTable++;\r
	358	}\r
	359	\r
1f3b1eb3 AB	360	TTBRAttributes = ArmHasMpExtensions () ? TTBR_MP_WRITE_BACK_ALLOC\r
1f3b1eb3 AB	361	: TTBR_WRITE_BACK_ALLOC;\r
d7f03464 AB	362	if (TTBRAttributes & TTBR_SHAREABLE) {\r
	363	if (PreferNonshareableMemory ()) {\r
	364	TTBRAttributes ^= TTBR_SHAREABLE;\r
	365	} else {\r
	366	//\r
	367	// Unlike the S bit in the short descriptors, which implies inner shareable\r
	368	// on an implementation that supports two levels, the meaning of the S bit\r
	369	// in the TTBR depends on the NOS bit, which defaults to Outer Shareable.\r
	370	// However, we should only set this bit after we have confirmed that the\r
	371	// implementation supports multiple levels, or else the NOS bit is UNK/SBZP\r
	372	//\r
	373	if (((ArmReadIdMmfr0 () >> 12) & 0xf) != 0) {\r
	374	TTBRAttributes \|= TTBR_NOT_OUTER_SHAREABLE;\r
	375	}\r
	376	}\r
	377	}\r
	378	\r
a17add32	379	ArmSetTTBR0 ((VOID *)((UINTN)TranslationTable \| TTBRAttributes));\r
d7f03464 AB	380	\r
	381	//\r
	382	// The TTBCR register value is undefined at reset in the Non-Secure world.\r
	383	// Writing 0 has the effect of:\r
	384	// Clearing EAE: Use short descriptors, as mandated by specification.\r
	385	// Clearing PD0 and PD1: Translation Table Walk Disable is off.\r
	386	// Clearing N: Perform all translation table walks through TTBR0.\r
	387	// (0 is the default reset value in systems not implementing\r
	388	// the Security Extensions.)\r
	389	//\r
	390	ArmSetTTBCR (0);\r
	391	\r
	392	ArmSetDomainAccessControl (DOMAIN_ACCESS_CONTROL_NONE(15) \|\r
	393	DOMAIN_ACCESS_CONTROL_NONE(14) \|\r
	394	DOMAIN_ACCESS_CONTROL_NONE(13) \|\r
	395	DOMAIN_ACCESS_CONTROL_NONE(12) \|\r
	396	DOMAIN_ACCESS_CONTROL_NONE(11) \|\r
	397	DOMAIN_ACCESS_CONTROL_NONE(10) \|\r
	398	DOMAIN_ACCESS_CONTROL_NONE( 9) \|\r
	399	DOMAIN_ACCESS_CONTROL_NONE( 8) \|\r
	400	DOMAIN_ACCESS_CONTROL_NONE( 7) \|\r
	401	DOMAIN_ACCESS_CONTROL_NONE( 6) \|\r
	402	DOMAIN_ACCESS_CONTROL_NONE( 5) \|\r
	403	DOMAIN_ACCESS_CONTROL_NONE( 4) \|\r
	404	DOMAIN_ACCESS_CONTROL_NONE( 3) \|\r
	405	DOMAIN_ACCESS_CONTROL_NONE( 2) \|\r
	406	DOMAIN_ACCESS_CONTROL_NONE( 1) \|\r
	407	DOMAIN_ACCESS_CONTROL_CLIENT(0));\r
	408	\r
	409	ArmEnableInstructionCache();\r
	410	ArmEnableDataCache();\r
	411	ArmEnableMmu();\r
	412	return RETURN_SUCCESS;\r
	413	}\r