[mirror_edk2.git] / ArmPkg / Library / ArmMmuLib / Arm / ArmMmuLibCore.c

/** @file\r
*  File managing the MMU for ARMv7 architecture\r
*\r
*  Copyright (c) 2011-2016, ARM Limited. All rights reserved.\r
*\r
*  SPDX-License-Identifier: BSD-2-Clause-Patent\r
*\r
**/\r
\r
#include <Uefi.h>\r
#include <Chipset/ArmV7.h>\r
#include <Library/BaseMemoryLib.h>\r
#include <Library/CacheMaintenanceLib.h>\r
#include <Library/MemoryAllocationLib.h>\r
#include <Library/ArmLib.h>\r
#include <Library/BaseLib.h>\r
#include <Library/DebugLib.h>\r
#include <Library/PcdLib.h>\r
\r
#define ID_MMFR0_SHARELVL_SHIFT  12\r
#define ID_MMFR0_SHARELVL_MASK   0xf\r
#define ID_MMFR0_SHARELVL_ONE    0\r
#define ID_MMFR0_SHARELVL_TWO    1\r
\r
#define ID_MMFR0_INNERSHR_SHIFT  28\r
#define ID_MMFR0_INNERSHR_MASK   0xf\r
#define ID_MMFR0_OUTERSHR_SHIFT  8\r
#define ID_MMFR0_OUTERSHR_MASK   0xf\r
\r
#define ID_MMFR0_SHR_IMP_UNCACHED     0\r
#define ID_MMFR0_SHR_IMP_HW_COHERENT  1\r
#define ID_MMFR0_SHR_IGNORED          0xf\r
\r
UINTN\r
EFIAPI\r
ArmReadIdMmfr0 (\r
  VOID\r
  );\r
\r
BOOLEAN\r
EFIAPI\r
ArmHasMpExtensions (\r
  VOID\r
  );\r
\r
STATIC\r
BOOLEAN\r
PreferNonshareableMemory (\r
  VOID\r
  )\r
{\r
  UINTN  Mmfr;\r
  UINTN  Val;\r
\r
  if (FeaturePcdGet (PcdNormalMemoryNonshareableOverride)) {\r
    return TRUE;\r
  }\r
\r
  //\r
  // Check whether the innermost level of shareability (the level we will use\r
  // by default to map normal memory) is implemented with hardware coherency\r
  // support. Otherwise, revert to mapping as non-shareable.\r
  //\r
  Mmfr = ArmReadIdMmfr0 ();\r
  switch ((Mmfr >> ID_MMFR0_SHARELVL_SHIFT) & ID_MMFR0_SHARELVL_MASK) {\r
    case ID_MMFR0_SHARELVL_ONE:\r
      // one level of shareability\r
      Val = (Mmfr >> ID_MMFR0_OUTERSHR_SHIFT) & ID_MMFR0_OUTERSHR_MASK;\r
      break;\r
    case ID_MMFR0_SHARELVL_TWO:\r
      // two levels of shareability\r
      Val = (Mmfr >> ID_MMFR0_INNERSHR_SHIFT) & ID_MMFR0_INNERSHR_MASK;\r
      break;\r
    default:\r
      // unexpected value -> shareable is the safe option\r
      ASSERT (FALSE);\r
      return FALSE;\r
  }\r
\r
  return Val != ID_MMFR0_SHR_IMP_HW_COHERENT;\r
}\r
\r
STATIC\r
VOID\r
PopulateLevel2PageTable (\r
  IN UINT32                        *SectionEntry,\r
  IN UINT32                        PhysicalBase,\r
  IN UINT32                        RemainLength,\r
  IN ARM_MEMORY_REGION_ATTRIBUTES  Attributes\r
  )\r
{\r
  UINT32  *PageEntry;\r
  UINT32  Pages;\r
  UINT32  Index;\r
  UINT32  PageAttributes;\r
  UINT32  SectionDescriptor;\r
  UINT32  TranslationTable;\r
  UINT32  BaseSectionAddress;\r
  UINT32  FirstPageOffset;\r
\r
  switch (Attributes) {\r
    case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK:\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK:\r
      PageAttributes = TT_DESCRIPTOR_PAGE_WRITE_BACK;\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK_NONSHAREABLE:\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK_NONSHAREABLE:\r
      PageAttributes  = TT_DESCRIPTOR_PAGE_WRITE_BACK;\r
      PageAttributes &= ~TT_DESCRIPTOR_PAGE_S_SHARED;\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_THROUGH:\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_THROUGH:\r
      PageAttributes = TT_DESCRIPTOR_PAGE_WRITE_THROUGH;\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_DEVICE:\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_DEVICE:\r
      PageAttributes = TT_DESCRIPTOR_PAGE_DEVICE;\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_UNCACHED_UNBUFFERED:\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_UNCACHED_UNBUFFERED:\r
      PageAttributes = TT_DESCRIPTOR_PAGE_UNCACHED;\r
      break;\r
    default:\r
      PageAttributes = TT_DESCRIPTOR_PAGE_UNCACHED;\r
      break;\r
  }\r
\r
  if (PreferNonshareableMemory ()) {\r
    PageAttributes &= ~TT_DESCRIPTOR_PAGE_S_SHARED;\r
  }\r
\r
  // Check if the Section Entry has already been populated. Otherwise attach a\r
  // Level 2 Translation Table to it\r
  if (*SectionEntry != 0) {\r
    // The entry must be a page table. Otherwise it exists an overlapping in the memory map\r
    if (TT_DESCRIPTOR_SECTION_TYPE_IS_PAGE_TABLE (*SectionEntry)) {\r
      TranslationTable = *SectionEntry & TT_DESCRIPTOR_SECTION_PAGETABLE_ADDRESS_MASK;\r
    } else if ((*SectionEntry & TT_DESCRIPTOR_SECTION_TYPE_MASK) == TT_DESCRIPTOR_SECTION_TYPE_SECTION) {\r
      // Case where a virtual memory map descriptor overlapped a section entry\r
\r
      // Allocate a Level2 Page Table for this Section\r
      TranslationTable = (UINTN)AllocateAlignedPages (\r
                                  EFI_SIZE_TO_PAGES (TRANSLATION_TABLE_PAGE_SIZE),\r
                                  TRANSLATION_TABLE_PAGE_ALIGNMENT\r
                                  );\r
\r
      // Translate the Section Descriptor into Page Descriptor\r
      SectionDescriptor = TT_DESCRIPTOR_PAGE_TYPE_PAGE | ConvertSectionAttributesToPageAttributes (*SectionEntry, FALSE);\r
\r
      BaseSectionAddress = TT_DESCRIPTOR_SECTION_BASE_ADDRESS (*SectionEntry);\r
\r
      //\r
      // Make sure we are not inadvertently hitting in the caches\r
      // when populating the page tables\r
      //\r
      InvalidateDataCacheRange (\r
        (VOID *)TranslationTable,\r
        TRANSLATION_TABLE_PAGE_SIZE\r
        );\r
\r
      // Populate the new Level2 Page Table for the section\r
      PageEntry = (UINT32 *)TranslationTable;\r
      for (Index = 0; Index < TRANSLATION_TABLE_PAGE_COUNT; Index++) {\r
        PageEntry[Index] = TT_DESCRIPTOR_PAGE_BASE_ADDRESS (BaseSectionAddress + (Index << 12)) | SectionDescriptor;\r
      }\r
\r
      // Overwrite the section entry to point to the new Level2 Translation Table\r
      *SectionEntry = (TranslationTable & TT_DESCRIPTOR_SECTION_PAGETABLE_ADDRESS_MASK) |\r
                      (IS_ARM_MEMORY_REGION_ATTRIBUTES_SECURE (Attributes) ? (1 << 3) : 0) |\r
                      TT_DESCRIPTOR_SECTION_TYPE_PAGE_TABLE;\r
    } else {\r
      // We do not support the other section type (16MB Section)\r
      ASSERT (0);\r
      return;\r
    }\r
  } else {\r
    TranslationTable = (UINTN)AllocateAlignedPages (\r
                                EFI_SIZE_TO_PAGES (TRANSLATION_TABLE_PAGE_SIZE),\r
                                TRANSLATION_TABLE_PAGE_ALIGNMENT\r
                                );\r
    //\r
    // Make sure we are not inadvertently hitting in the caches\r
    // when populating the page tables\r
    //\r
    InvalidateDataCacheRange (\r
      (VOID *)TranslationTable,\r
      TRANSLATION_TABLE_PAGE_SIZE\r
      );\r
    ZeroMem ((VOID *)TranslationTable, TRANSLATION_TABLE_PAGE_SIZE);\r
\r
    *SectionEntry = (TranslationTable & TT_DESCRIPTOR_SECTION_PAGETABLE_ADDRESS_MASK) |\r
                    (IS_ARM_MEMORY_REGION_ATTRIBUTES_SECURE (Attributes) ? (1 << 3) : 0) |\r
                    TT_DESCRIPTOR_SECTION_TYPE_PAGE_TABLE;\r
  }\r
\r
  FirstPageOffset = (PhysicalBase & TT_DESCRIPTOR_PAGE_INDEX_MASK) >> TT_DESCRIPTOR_PAGE_BASE_SHIFT;\r
  PageEntry       = (UINT32 *)TranslationTable + FirstPageOffset;\r
  Pages           = RemainLength / TT_DESCRIPTOR_PAGE_SIZE;\r
\r
  ASSERT (FirstPageOffset + Pages <= TRANSLATION_TABLE_PAGE_COUNT);\r
\r
  for (Index = 0; Index < Pages; Index++) {\r
    *PageEntry++  =  TT_DESCRIPTOR_PAGE_BASE_ADDRESS (PhysicalBase) | PageAttributes;\r
    PhysicalBase += TT_DESCRIPTOR_PAGE_SIZE;\r
  }\r
\r
  //\r
  // Invalidate again to ensure that any line fetches that may have occurred\r
  // [speculatively] since the previous invalidate are evicted again.\r
  //\r
  ArmDataMemoryBarrier ();\r
  InvalidateDataCacheRange (\r
    (UINT32 *)TranslationTable + FirstPageOffset,\r
    RemainLength / TT_DESCRIPTOR_PAGE_SIZE * sizeof (*PageEntry)\r
    );\r
}\r
\r
STATIC\r
VOID\r
FillTranslationTable (\r
  IN  UINT32                        *TranslationTable,\r
  IN  ARM_MEMORY_REGION_DESCRIPTOR  *MemoryRegion\r
  )\r
{\r
  UINT32  *SectionEntry;\r
  UINT32  Attributes;\r
  UINT32  PhysicalBase;\r
  UINT64  RemainLength;\r
  UINT32  PageMapLength;\r
\r
  ASSERT (MemoryRegion->Length > 0);\r
\r
  if (MemoryRegion->PhysicalBase >= SIZE_4GB) {\r
    return;\r
  }\r
\r
  PhysicalBase = (UINT32)MemoryRegion->PhysicalBase;\r
  RemainLength = MIN (MemoryRegion->Length, SIZE_4GB - PhysicalBase);\r
\r
  switch (MemoryRegion->Attributes) {\r
    case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK:\r
      Attributes = TT_DESCRIPTOR_SECTION_WRITE_BACK (0);\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK_NONSHAREABLE:\r
      Attributes  = TT_DESCRIPTOR_SECTION_WRITE_BACK (0);\r
      Attributes &= ~TT_DESCRIPTOR_SECTION_S_SHARED;\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_THROUGH:\r
      Attributes = TT_DESCRIPTOR_SECTION_WRITE_THROUGH (0);\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_DEVICE:\r
      Attributes = TT_DESCRIPTOR_SECTION_DEVICE (0);\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_UNCACHED_UNBUFFERED:\r
      Attributes = TT_DESCRIPTOR_SECTION_UNCACHED (0);\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK:\r
      Attributes = TT_DESCRIPTOR_SECTION_WRITE_BACK (1);\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK_NONSHAREABLE:\r
      Attributes  = TT_DESCRIPTOR_SECTION_WRITE_BACK (1);\r
      Attributes &= ~TT_DESCRIPTOR_SECTION_S_SHARED;\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_THROUGH:\r
      Attributes = TT_DESCRIPTOR_SECTION_WRITE_THROUGH (1);\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_DEVICE:\r
      Attributes = TT_DESCRIPTOR_SECTION_DEVICE (1);\r
      break;\r
    case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_UNCACHED_UNBUFFERED:\r
      Attributes = TT_DESCRIPTOR_SECTION_UNCACHED (1);\r
      break;\r
    default:\r
      Attributes = TT_DESCRIPTOR_SECTION_UNCACHED (0);\r
      break;\r
  }\r
\r
  if (PreferNonshareableMemory ()) {\r
    Attributes &= ~TT_DESCRIPTOR_SECTION_S_SHARED;\r
  }\r
\r
  // Get the first section entry for this mapping\r
  SectionEntry = TRANSLATION_TABLE_ENTRY_FOR_VIRTUAL_ADDRESS (TranslationTable, MemoryRegion->VirtualBase);\r
\r
  while (RemainLength != 0) {\r
    if ((PhysicalBase % TT_DESCRIPTOR_SECTION_SIZE == 0) &&\r
        (RemainLength >= TT_DESCRIPTOR_SECTION_SIZE))\r
    {\r
      // Case: Physical address aligned on the Section Size (1MB) && the length\r
      // is greater than the Section Size\r
      *SectionEntry = TT_DESCRIPTOR_SECTION_BASE_ADDRESS (PhysicalBase) | Attributes;\r
\r
      //\r
      // Issue a DMB to ensure that the page table entry update made it to\r
      // memory before we issue the invalidate, otherwise, a subsequent\r
      // speculative fetch could observe the old value.\r
      //\r
      ArmDataMemoryBarrier ();\r
      ArmInvalidateDataCacheEntryByMVA ((UINTN)SectionEntry++);\r
\r
      PhysicalBase += TT_DESCRIPTOR_SECTION_SIZE;\r
      RemainLength -= TT_DESCRIPTOR_SECTION_SIZE;\r
    } else {\r
      PageMapLength = MIN (\r
                        (UINT32)RemainLength,\r
                        TT_DESCRIPTOR_SECTION_SIZE -\r
                        (PhysicalBase % TT_DESCRIPTOR_SECTION_SIZE)\r
                        );\r
\r
      // Case: Physical address aligned on the Section Size (1MB) && the length\r
      //       does not fill a section\r
      // Case: Physical address NOT aligned on the Section Size (1MB)\r
      PopulateLevel2PageTable (\r
        SectionEntry,\r
        PhysicalBase,\r
        PageMapLength,\r
        MemoryRegion->Attributes\r
        );\r
\r
      //\r
      // Issue a DMB to ensure that the page table entry update made it to\r
      // memory before we issue the invalidate, otherwise, a subsequent\r
      // speculative fetch could observe the old value.\r
      //\r
      ArmDataMemoryBarrier ();\r
      ArmInvalidateDataCacheEntryByMVA ((UINTN)SectionEntry++);\r
\r
      // If it is the last entry\r
      if (RemainLength < TT_DESCRIPTOR_SECTION_SIZE) {\r
        break;\r
      }\r
\r
      PhysicalBase += PageMapLength;\r
      RemainLength -= PageMapLength;\r
    }\r
  }\r
}\r
\r
RETURN_STATUS\r
EFIAPI\r
ArmConfigureMmu (\r
  IN  ARM_MEMORY_REGION_DESCRIPTOR  *MemoryTable,\r
  OUT VOID                          **TranslationTableBase OPTIONAL,\r
  OUT UINTN                         *TranslationTableSize OPTIONAL\r
  )\r
{\r
  VOID    *TranslationTable;\r
  UINT32  TTBRAttributes;\r
\r
  TranslationTable = AllocateAlignedPages (\r
                       EFI_SIZE_TO_PAGES (TRANSLATION_TABLE_SECTION_SIZE),\r
                       TRANSLATION_TABLE_SECTION_ALIGNMENT\r
                       );\r
  if (TranslationTable == NULL) {\r
    return RETURN_OUT_OF_RESOURCES;\r
  }\r
\r
  if (TranslationTableBase != NULL) {\r
    *TranslationTableBase = TranslationTable;\r
  }\r
\r
  if (TranslationTableSize != NULL) {\r
    *TranslationTableSize = TRANSLATION_TABLE_SECTION_SIZE;\r
  }\r
\r
  //\r
  // Make sure we are not inadvertently hitting in the caches\r
  // when populating the page tables\r
  //\r
  InvalidateDataCacheRange (TranslationTable, TRANSLATION_TABLE_SECTION_SIZE);\r
  ZeroMem (TranslationTable, TRANSLATION_TABLE_SECTION_SIZE);\r
\r
  while (MemoryTable->Length != 0) {\r
    FillTranslationTable (TranslationTable, MemoryTable);\r
    MemoryTable++;\r
  }\r
\r
  TTBRAttributes = ArmHasMpExtensions () ? TTBR_MP_WRITE_BACK_ALLOC\r
                                         : TTBR_WRITE_BACK_ALLOC;\r
  if (TTBRAttributes & TTBR_SHAREABLE) {\r
    if (PreferNonshareableMemory ()) {\r
      TTBRAttributes ^= TTBR_SHAREABLE;\r
    } else {\r
      //\r
      // Unlike the S bit in the short descriptors, which implies inner shareable\r
      // on an implementation that supports two levels, the meaning of the S bit\r
      // in the TTBR depends on the NOS bit, which defaults to Outer Shareable.\r
      // However, we should only set this bit after we have confirmed that the\r
      // implementation supports multiple levels, or else the NOS bit is UNK/SBZP\r
      //\r
      if (((ArmReadIdMmfr0 () >> 12) & 0xf) != 0) {\r
        TTBRAttributes |= TTBR_NOT_OUTER_SHAREABLE;\r
      }\r
    }\r
  }\r
\r
  ArmSetTTBR0 ((VOID *)((UINTN)TranslationTable | TTBRAttributes));\r
\r
  //\r
  // The TTBCR register value is undefined at reset in the Non-Secure world.\r
  // Writing 0 has the effect of:\r
  //   Clearing EAE: Use short descriptors, as mandated by specification.\r
  //   Clearing PD0 and PD1: Translation Table Walk Disable is off.\r
  //   Clearing N: Perform all translation table walks through TTBR0.\r
  //               (0 is the default reset value in systems not implementing\r
  //               the Security Extensions.)\r
  //\r
  ArmSetTTBCR (0);\r
\r
  ArmSetDomainAccessControl (\r
    DOMAIN_ACCESS_CONTROL_NONE (15) |\r
    DOMAIN_ACCESS_CONTROL_NONE (14) |\r
    DOMAIN_ACCESS_CONTROL_NONE (13) |\r
    DOMAIN_ACCESS_CONTROL_NONE (12) |\r
    DOMAIN_ACCESS_CONTROL_NONE (11) |\r
    DOMAIN_ACCESS_CONTROL_NONE (10) |\r
    DOMAIN_ACCESS_CONTROL_NONE (9) |\r
    DOMAIN_ACCESS_CONTROL_NONE (8) |\r
    DOMAIN_ACCESS_CONTROL_NONE (7) |\r
    DOMAIN_ACCESS_CONTROL_NONE (6) |\r
    DOMAIN_ACCESS_CONTROL_NONE (5) |\r
    DOMAIN_ACCESS_CONTROL_NONE (4) |\r
    DOMAIN_ACCESS_CONTROL_NONE (3) |\r
    DOMAIN_ACCESS_CONTROL_NONE (2) |\r
    DOMAIN_ACCESS_CONTROL_NONE (1) |\r
    DOMAIN_ACCESS_CONTROL_CLIENT (0)\r
    );\r
\r
  ArmEnableInstructionCache ();\r
  ArmEnableDataCache ();\r
  ArmEnableMmu ();\r
  return RETURN_SUCCESS;\r
}\r
Commit	Line	Data
d7f03464 AB	1	/** @file\r
	2	* File managing the MMU for ARMv7 architecture\r
	3	*\r
	4	* Copyright (c) 2011-2016, ARM Limited. All rights reserved.\r
	5	*\r
4059386c	6	* SPDX-License-Identifier: BSD-2-Clause-Patent\r
d7f03464 AB	7	*\r
	8	**/\r
	9	\r
	10	#include <Uefi.h>\r
	11	#include <Chipset/ArmV7.h>\r
	12	#include <Library/BaseMemoryLib.h>\r
521f3ced	13	#include <Library/CacheMaintenanceLib.h>\r
d7f03464 AB	14	#include <Library/MemoryAllocationLib.h>\r
	15	#include <Library/ArmLib.h>\r
	16	#include <Library/BaseLib.h>\r
	17	#include <Library/DebugLib.h>\r
	18	#include <Library/PcdLib.h>\r
	19	\r
429309e0 MK	20	#define ID_MMFR0_SHARELVL_SHIFT 12\r
	21	#define ID_MMFR0_SHARELVL_MASK 0xf\r
	22	#define ID_MMFR0_SHARELVL_ONE 0\r
	23	#define ID_MMFR0_SHARELVL_TWO 1\r
d7f03464	24	\r
429309e0 MK	25	#define ID_MMFR0_INNERSHR_SHIFT 28\r
	26	#define ID_MMFR0_INNERSHR_MASK 0xf\r
	27	#define ID_MMFR0_OUTERSHR_SHIFT 8\r
	28	#define ID_MMFR0_OUTERSHR_MASK 0xf\r
d7f03464	29	\r
429309e0 MK	30	#define ID_MMFR0_SHR_IMP_UNCACHED 0\r
	31	#define ID_MMFR0_SHR_IMP_HW_COHERENT 1\r
	32	#define ID_MMFR0_SHR_IGNORED 0xf\r
d7f03464 AB	33	\r
	34	UINTN\r
	35	EFIAPI\r
	36	ArmReadIdMmfr0 (\r
	37	VOID\r
	38	);\r
	39	\r
	40	BOOLEAN\r
	41	EFIAPI\r
	42	ArmHasMpExtensions (\r
	43	VOID\r
	44	);\r
	45	\r
d7f03464 AB	46	STATIC\r
	47	BOOLEAN\r
	48	PreferNonshareableMemory (\r
	49	VOID\r
	50	)\r
	51	{\r
429309e0 MK	52	UINTN Mmfr;\r
429309e0 MK	53	UINTN Val;\r
d7f03464 AB	54	\r
	55	if (FeaturePcdGet (PcdNormalMemoryNonshareableOverride)) {\r
	56	return TRUE;\r
	57	}\r
	58	\r
	59	//\r
	60	// Check whether the innermost level of shareability (the level we will use\r
	61	// by default to map normal memory) is implemented with hardware coherency\r
	62	// support. Otherwise, revert to mapping as non-shareable.\r
	63	//\r
	64	Mmfr = ArmReadIdMmfr0 ();\r
	65	switch ((Mmfr >> ID_MMFR0_SHARELVL_SHIFT) & ID_MMFR0_SHARELVL_MASK) {\r
429309e0 MK	66	case ID_MMFR0_SHARELVL_ONE:\r
	67	// one level of shareability\r
	68	Val = (Mmfr >> ID_MMFR0_OUTERSHR_SHIFT) & ID_MMFR0_OUTERSHR_MASK;\r
	69	break;\r
	70	case ID_MMFR0_SHARELVL_TWO:\r
	71	// two levels of shareability\r
	72	Val = (Mmfr >> ID_MMFR0_INNERSHR_SHIFT) & ID_MMFR0_INNERSHR_MASK;\r
	73	break;\r
	74	default:\r
	75	// unexpected value -> shareable is the safe option\r
	76	ASSERT (FALSE);\r
	77	return FALSE;\r
d7f03464	78	}\r
429309e0	79	\r
d7f03464 AB	80	return Val != ID_MMFR0_SHR_IMP_HW_COHERENT;\r
	81	}\r
	82	\r
	83	STATIC\r
	84	VOID\r
	85	PopulateLevel2PageTable (\r
429309e0 MK	86	IN UINT32 *SectionEntry,\r
	87	IN UINT32 PhysicalBase,\r
	88	IN UINT32 RemainLength,\r
	89	IN ARM_MEMORY_REGION_ATTRIBUTES Attributes\r
d7f03464 AB	90	)\r
d7f03464 AB	91	{\r
429309e0	92	UINT32 *PageEntry;\r
d7f03464 AB	93	UINT32 Pages;\r
	94	UINT32 Index;\r
	95	UINT32 PageAttributes;\r
	96	UINT32 SectionDescriptor;\r
	97	UINT32 TranslationTable;\r
	98	UINT32 BaseSectionAddress;\r
889c7ca1	99	UINT32 FirstPageOffset;\r
d7f03464 AB	100	\r
	101	switch (Attributes) {\r
	102	case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK:\r
	103	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK:\r
	104	PageAttributes = TT_DESCRIPTOR_PAGE_WRITE_BACK;\r
	105	break;\r
e3ad54fa AB	106	case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK_NONSHAREABLE:\r
e3ad54fa AB	107	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK_NONSHAREABLE:\r
429309e0	108	PageAttributes = TT_DESCRIPTOR_PAGE_WRITE_BACK;\r
e3ad54fa AB	109	PageAttributes &= ~TT_DESCRIPTOR_PAGE_S_SHARED;\r
e3ad54fa AB	110	break;\r
d7f03464 AB	111	case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_THROUGH:\r
	112	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_THROUGH:\r
	113	PageAttributes = TT_DESCRIPTOR_PAGE_WRITE_THROUGH;\r
	114	break;\r
	115	case ARM_MEMORY_REGION_ATTRIBUTE_DEVICE:\r
	116	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_DEVICE:\r
	117	PageAttributes = TT_DESCRIPTOR_PAGE_DEVICE;\r
	118	break;\r
	119	case ARM_MEMORY_REGION_ATTRIBUTE_UNCACHED_UNBUFFERED:\r
	120	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_UNCACHED_UNBUFFERED:\r
	121	PageAttributes = TT_DESCRIPTOR_PAGE_UNCACHED;\r
	122	break;\r
	123	default:\r
	124	PageAttributes = TT_DESCRIPTOR_PAGE_UNCACHED;\r
	125	break;\r
	126	}\r
	127	\r
	128	if (PreferNonshareableMemory ()) {\r
	129	PageAttributes &= ~TT_DESCRIPTOR_PAGE_S_SHARED;\r
	130	}\r
	131	\r
	132	// Check if the Section Entry has already been populated. Otherwise attach a\r
	133	// Level 2 Translation Table to it\r
	134	if (*SectionEntry != 0) {\r
	135	// The entry must be a page table. Otherwise it exists an overlapping in the memory map\r
429309e0	136	if (TT_DESCRIPTOR_SECTION_TYPE_IS_PAGE_TABLE (*SectionEntry)) {\r
d7f03464 AB	137	TranslationTable = *SectionEntry & TT_DESCRIPTOR_SECTION_PAGETABLE_ADDRESS_MASK;\r
	138	} else if ((*SectionEntry & TT_DESCRIPTOR_SECTION_TYPE_MASK) == TT_DESCRIPTOR_SECTION_TYPE_SECTION) {\r
	139	// Case where a virtual memory map descriptor overlapped a section entry\r
	140	\r
	141	// Allocate a Level2 Page Table for this Section\r
825c3e2c AB	142	TranslationTable = (UINTN)AllocateAlignedPages (\r
825c3e2c AB	143	EFI_SIZE_TO_PAGES (TRANSLATION_TABLE_PAGE_SIZE),\r
429309e0 MK	144	TRANSLATION_TABLE_PAGE_ALIGNMENT\r
429309e0 MK	145	);\r
d7f03464 AB	146	\r
	147	// Translate the Section Descriptor into Page Descriptor\r
	148	SectionDescriptor = TT_DESCRIPTOR_PAGE_TYPE_PAGE \| ConvertSectionAttributesToPageAttributes (*SectionEntry, FALSE);\r
	149	\r
429309e0	150	BaseSectionAddress = TT_DESCRIPTOR_SECTION_BASE_ADDRESS (*SectionEntry);\r
d7f03464	151	\r
02d7797d AB	152	//\r
	153	// Make sure we are not inadvertently hitting in the caches\r
	154	// when populating the page tables\r
	155	//\r
429309e0 MK	156	InvalidateDataCacheRange (\r
	157	(VOID *)TranslationTable,\r
	158	TRANSLATION_TABLE_PAGE_SIZE\r
	159	);\r
02d7797d	160	\r
d7f03464	161	// Populate the new Level2 Page Table for the section\r
429309e0	162	PageEntry = (UINT32 *)TranslationTable;\r
d7f03464	163	for (Index = 0; Index < TRANSLATION_TABLE_PAGE_COUNT; Index++) {\r
429309e0	164	PageEntry[Index] = TT_DESCRIPTOR_PAGE_BASE_ADDRESS (BaseSectionAddress + (Index << 12)) \| SectionDescriptor;\r
d7f03464 AB	165	}\r
	166	\r
	167	// Overwrite the section entry to point to the new Level2 Translation Table\r
	168	*SectionEntry = (TranslationTable & TT_DESCRIPTOR_SECTION_PAGETABLE_ADDRESS_MASK) \|\r
429309e0 MK	169	(IS_ARM_MEMORY_REGION_ATTRIBUTES_SECURE (Attributes) ? (1 << 3) : 0) \|\r
429309e0 MK	170	TT_DESCRIPTOR_SECTION_TYPE_PAGE_TABLE;\r
d7f03464 AB	171	} else {\r
d7f03464 AB	172	// We do not support the other section type (16MB Section)\r
429309e0	173	ASSERT (0);\r
d7f03464 AB	174	return;\r
	175	}\r
	176	} else {\r
825c3e2c AB	177	TranslationTable = (UINTN)AllocateAlignedPages (\r
825c3e2c AB	178	EFI_SIZE_TO_PAGES (TRANSLATION_TABLE_PAGE_SIZE),\r
429309e0 MK	179	TRANSLATION_TABLE_PAGE_ALIGNMENT\r
429309e0 MK	180	);\r
02d7797d AB	181	//\r
	182	// Make sure we are not inadvertently hitting in the caches\r
	183	// when populating the page tables\r
	184	//\r
429309e0 MK	185	InvalidateDataCacheRange (\r
	186	(VOID *)TranslationTable,\r
	187	TRANSLATION_TABLE_PAGE_SIZE\r
	188	);\r
d7f03464 AB	189	ZeroMem ((VOID *)TranslationTable, TRANSLATION_TABLE_PAGE_SIZE);\r
	190	\r
	191	*SectionEntry = (TranslationTable & TT_DESCRIPTOR_SECTION_PAGETABLE_ADDRESS_MASK) \|\r
429309e0 MK	192	(IS_ARM_MEMORY_REGION_ATTRIBUTES_SECURE (Attributes) ? (1 << 3) : 0) \|\r
429309e0 MK	193	TT_DESCRIPTOR_SECTION_TYPE_PAGE_TABLE;\r
d7f03464 AB	194	}\r
d7f03464 AB	195	\r
889c7ca1	196	FirstPageOffset = (PhysicalBase & TT_DESCRIPTOR_PAGE_INDEX_MASK) >> TT_DESCRIPTOR_PAGE_BASE_SHIFT;\r
429309e0 MK	197	PageEntry = (UINT32 *)TranslationTable + FirstPageOffset;\r
429309e0 MK	198	Pages = RemainLength / TT_DESCRIPTOR_PAGE_SIZE;\r
d7f03464	199	\r
889c7ca1 MZ	200	ASSERT (FirstPageOffset + Pages <= TRANSLATION_TABLE_PAGE_COUNT);\r
889c7ca1 MZ	201	\r
d7f03464	202	for (Index = 0; Index < Pages; Index++) {\r
429309e0	203	*PageEntry++ = TT_DESCRIPTOR_PAGE_BASE_ADDRESS (PhysicalBase) \| PageAttributes;\r
d7f03464 AB	204	PhysicalBase += TT_DESCRIPTOR_PAGE_SIZE;\r
	205	}\r
	206	\r
02d7797d AB	207	//\r
	208	// Invalidate again to ensure that any line fetches that may have occurred\r
	209	// [speculatively] since the previous invalidate are evicted again.\r
	210	//\r
	211	ArmDataMemoryBarrier ();\r
429309e0 MK	212	InvalidateDataCacheRange (\r
	213	(UINT32 *)TranslationTable + FirstPageOffset,\r
	214	RemainLength / TT_DESCRIPTOR_PAGE_SIZE * sizeof (*PageEntry)\r
	215	);\r
d7f03464 AB	216	}\r
	217	\r
	218	STATIC\r
	219	VOID\r
	220	FillTranslationTable (\r
	221	IN UINT32 *TranslationTable,\r
	222	IN ARM_MEMORY_REGION_DESCRIPTOR *MemoryRegion\r
	223	)\r
	224	{\r
	225	UINT32 *SectionEntry;\r
	226	UINT32 Attributes;\r
	227	UINT32 PhysicalBase;\r
	228	UINT64 RemainLength;\r
889c7ca1	229	UINT32 PageMapLength;\r
d7f03464	230	\r
429309e0	231	ASSERT (MemoryRegion->Length > 0);\r
d7f03464 AB	232	\r
	233	if (MemoryRegion->PhysicalBase >= SIZE_4GB) {\r
	234	return;\r
	235	}\r
	236	\r
2bdc9042	237	PhysicalBase = (UINT32)MemoryRegion->PhysicalBase;\r
429309e0	238	RemainLength = MIN (MemoryRegion->Length, SIZE_4GB - PhysicalBase);\r
d7f03464 AB	239	\r
	240	switch (MemoryRegion->Attributes) {\r
	241	case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK:\r
429309e0	242	Attributes = TT_DESCRIPTOR_SECTION_WRITE_BACK (0);\r
d7f03464	243	break;\r
e3ad54fa	244	case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK_NONSHAREABLE:\r
429309e0	245	Attributes = TT_DESCRIPTOR_SECTION_WRITE_BACK (0);\r
e3ad54fa AB	246	Attributes &= ~TT_DESCRIPTOR_SECTION_S_SHARED;\r
e3ad54fa AB	247	break;\r
d7f03464	248	case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_THROUGH:\r
429309e0	249	Attributes = TT_DESCRIPTOR_SECTION_WRITE_THROUGH (0);\r
d7f03464 AB	250	break;\r
d7f03464 AB	251	case ARM_MEMORY_REGION_ATTRIBUTE_DEVICE:\r
429309e0	252	Attributes = TT_DESCRIPTOR_SECTION_DEVICE (0);\r
d7f03464 AB	253	break;\r
d7f03464 AB	254	case ARM_MEMORY_REGION_ATTRIBUTE_UNCACHED_UNBUFFERED:\r
429309e0	255	Attributes = TT_DESCRIPTOR_SECTION_UNCACHED (0);\r
d7f03464 AB	256	break;\r
d7f03464 AB	257	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK:\r
429309e0	258	Attributes = TT_DESCRIPTOR_SECTION_WRITE_BACK (1);\r
d7f03464	259	break;\r
e3ad54fa	260	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_BACK_NONSHAREABLE:\r
429309e0	261	Attributes = TT_DESCRIPTOR_SECTION_WRITE_BACK (1);\r
e3ad54fa AB	262	Attributes &= ~TT_DESCRIPTOR_SECTION_S_SHARED;\r
e3ad54fa AB	263	break;\r
d7f03464	264	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_WRITE_THROUGH:\r
429309e0	265	Attributes = TT_DESCRIPTOR_SECTION_WRITE_THROUGH (1);\r
d7f03464 AB	266	break;\r
d7f03464 AB	267	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_DEVICE:\r
429309e0	268	Attributes = TT_DESCRIPTOR_SECTION_DEVICE (1);\r
d7f03464 AB	269	break;\r
d7f03464 AB	270	case ARM_MEMORY_REGION_ATTRIBUTE_NONSECURE_UNCACHED_UNBUFFERED:\r
429309e0	271	Attributes = TT_DESCRIPTOR_SECTION_UNCACHED (1);\r
d7f03464 AB	272	break;\r
d7f03464 AB	273	default:\r
429309e0	274	Attributes = TT_DESCRIPTOR_SECTION_UNCACHED (0);\r
d7f03464 AB	275	break;\r
	276	}\r
	277	\r
	278	if (PreferNonshareableMemory ()) {\r
	279	Attributes &= ~TT_DESCRIPTOR_SECTION_S_SHARED;\r
	280	}\r
	281	\r
	282	// Get the first section entry for this mapping\r
429309e0	283	SectionEntry = TRANSLATION_TABLE_ENTRY_FOR_VIRTUAL_ADDRESS (TranslationTable, MemoryRegion->VirtualBase);\r
d7f03464 AB	284	\r
d7f03464 AB	285	while (RemainLength != 0) {\r
429309e0 MK	286	if ((PhysicalBase % TT_DESCRIPTOR_SECTION_SIZE == 0) &&\r
	287	(RemainLength >= TT_DESCRIPTOR_SECTION_SIZE))\r
	288	{\r
889c7ca1 MZ	289	// Case: Physical address aligned on the Section Size (1MB) && the length\r
889c7ca1 MZ	290	// is greater than the Section Size\r
429309e0	291	*SectionEntry = TT_DESCRIPTOR_SECTION_BASE_ADDRESS (PhysicalBase) \| Attributes;\r
02d7797d AB	292	\r
	293	//\r
	294	// Issue a DMB to ensure that the page table entry update made it to\r
	295	// memory before we issue the invalidate, otherwise, a subsequent\r
	296	// speculative fetch could observe the old value.\r
	297	//\r
	298	ArmDataMemoryBarrier ();\r
	299	ArmInvalidateDataCacheEntryByMVA ((UINTN)SectionEntry++);\r
	300	\r
889c7ca1 MZ	301	PhysicalBase += TT_DESCRIPTOR_SECTION_SIZE;\r
889c7ca1 MZ	302	RemainLength -= TT_DESCRIPTOR_SECTION_SIZE;\r
d7f03464	303	} else {\r
429309e0 MK	304	PageMapLength = MIN (\r
	305	(UINT32)RemainLength,\r
	306	TT_DESCRIPTOR_SECTION_SIZE -\r
	307	(PhysicalBase % TT_DESCRIPTOR_SECTION_SIZE)\r
	308	);\r
889c7ca1 MZ	309	\r
	310	// Case: Physical address aligned on the Section Size (1MB) && the length\r
	311	// does not fill a section\r
d7f03464	312	// Case: Physical address NOT aligned on the Section Size (1MB)\r
429309e0 MK	313	PopulateLevel2PageTable (\r
	314	SectionEntry,\r
	315	PhysicalBase,\r
	316	PageMapLength,\r
	317	MemoryRegion->Attributes\r
	318	);\r
d7f03464	319	\r
02d7797d AB	320	//\r
	321	// Issue a DMB to ensure that the page table entry update made it to\r
	322	// memory before we issue the invalidate, otherwise, a subsequent\r
	323	// speculative fetch could observe the old value.\r
	324	//\r
	325	ArmDataMemoryBarrier ();\r
	326	ArmInvalidateDataCacheEntryByMVA ((UINTN)SectionEntry++);\r
	327	\r
d7f03464 AB	328	// If it is the last entry\r
	329	if (RemainLength < TT_DESCRIPTOR_SECTION_SIZE) {\r
	330	break;\r
	331	}\r
889c7ca1 MZ	332	\r
	333	PhysicalBase += PageMapLength;\r
	334	RemainLength -= PageMapLength;\r
d7f03464	335	}\r
d7f03464 AB	336	}\r
	337	}\r
	338	\r
	339	RETURN_STATUS\r
	340	EFIAPI\r
	341	ArmConfigureMmu (\r
	342	IN ARM_MEMORY_REGION_DESCRIPTOR *MemoryTable,\r
429309e0	343	OUT VOID **TranslationTableBase OPTIONAL,\r
d7f03464 AB	344	OUT UINTN *TranslationTableSize OPTIONAL\r
	345	)\r
	346	{\r
429309e0 MK	347	VOID *TranslationTable;\r
429309e0 MK	348	UINT32 TTBRAttributes;\r
d7f03464	349	\r
825c3e2c AB	350	TranslationTable = AllocateAlignedPages (\r
825c3e2c AB	351	EFI_SIZE_TO_PAGES (TRANSLATION_TABLE_SECTION_SIZE),\r
429309e0 MK	352	TRANSLATION_TABLE_SECTION_ALIGNMENT\r
429309e0 MK	353	);\r
d7f03464 AB	354	if (TranslationTable == NULL) {\r
	355	return RETURN_OUT_OF_RESOURCES;\r
	356	}\r
d7f03464 AB	357	\r
	358	if (TranslationTableBase != NULL) {\r
	359	*TranslationTableBase = TranslationTable;\r
	360	}\r
	361	\r
	362	if (TranslationTableSize != NULL) {\r
	363	*TranslationTableSize = TRANSLATION_TABLE_SECTION_SIZE;\r
	364	}\r
	365	\r
02d7797d AB	366	//\r
	367	// Make sure we are not inadvertently hitting in the caches\r
	368	// when populating the page tables\r
	369	//\r
	370	InvalidateDataCacheRange (TranslationTable, TRANSLATION_TABLE_SECTION_SIZE);\r
d7f03464 AB	371	ZeroMem (TranslationTable, TRANSLATION_TABLE_SECTION_SIZE);\r
d7f03464 AB	372	\r
d7f03464	373	while (MemoryTable->Length != 0) {\r
d7f03464 AB	374	FillTranslationTable (TranslationTable, MemoryTable);\r
	375	MemoryTable++;\r
	376	}\r
	377	\r
1f3b1eb3 AB	378	TTBRAttributes = ArmHasMpExtensions () ? TTBR_MP_WRITE_BACK_ALLOC\r
1f3b1eb3 AB	379	: TTBR_WRITE_BACK_ALLOC;\r
d7f03464 AB	380	if (TTBRAttributes & TTBR_SHAREABLE) {\r
	381	if (PreferNonshareableMemory ()) {\r
	382	TTBRAttributes ^= TTBR_SHAREABLE;\r
	383	} else {\r
	384	//\r
	385	// Unlike the S bit in the short descriptors, which implies inner shareable\r
	386	// on an implementation that supports two levels, the meaning of the S bit\r
	387	// in the TTBR depends on the NOS bit, which defaults to Outer Shareable.\r
	388	// However, we should only set this bit after we have confirmed that the\r
	389	// implementation supports multiple levels, or else the NOS bit is UNK/SBZP\r
	390	//\r
	391	if (((ArmReadIdMmfr0 () >> 12) & 0xf) != 0) {\r
	392	TTBRAttributes \|= TTBR_NOT_OUTER_SHAREABLE;\r
	393	}\r
	394	}\r
	395	}\r
	396	\r
a17add32	397	ArmSetTTBR0 ((VOID *)((UINTN)TranslationTable \| TTBRAttributes));\r
d7f03464 AB	398	\r
	399	//\r
	400	// The TTBCR register value is undefined at reset in the Non-Secure world.\r
	401	// Writing 0 has the effect of:\r
	402	// Clearing EAE: Use short descriptors, as mandated by specification.\r
	403	// Clearing PD0 and PD1: Translation Table Walk Disable is off.\r
	404	// Clearing N: Perform all translation table walks through TTBR0.\r
	405	// (0 is the default reset value in systems not implementing\r
	406	// the Security Extensions.)\r
	407	//\r
	408	ArmSetTTBCR (0);\r
	409	\r
429309e0 MK	410	ArmSetDomainAccessControl (\r
	411	DOMAIN_ACCESS_CONTROL_NONE (15) \|\r
	412	DOMAIN_ACCESS_CONTROL_NONE (14) \|\r
	413	DOMAIN_ACCESS_CONTROL_NONE (13) \|\r
	414	DOMAIN_ACCESS_CONTROL_NONE (12) \|\r
	415	DOMAIN_ACCESS_CONTROL_NONE (11) \|\r
	416	DOMAIN_ACCESS_CONTROL_NONE (10) \|\r
	417	DOMAIN_ACCESS_CONTROL_NONE (9) \|\r
	418	DOMAIN_ACCESS_CONTROL_NONE (8) \|\r
	419	DOMAIN_ACCESS_CONTROL_NONE (7) \|\r
	420	DOMAIN_ACCESS_CONTROL_NONE (6) \|\r
	421	DOMAIN_ACCESS_CONTROL_NONE (5) \|\r
	422	DOMAIN_ACCESS_CONTROL_NONE (4) \|\r
	423	DOMAIN_ACCESS_CONTROL_NONE (3) \|\r
	424	DOMAIN_ACCESS_CONTROL_NONE (2) \|\r
	425	DOMAIN_ACCESS_CONTROL_NONE (1) \|\r
	426	DOMAIN_ACCESS_CONTROL_CLIENT (0)\r
	427	);\r
	428	\r
	429	ArmEnableInstructionCache ();\r
	430	ArmEnableDataCache ();\r
	431	ArmEnableMmu ();\r
d7f03464 AB	432	return RETURN_SUCCESS;\r
d7f03464 AB	433	}\r