[mirror_edk2.git] / IntelSiliconPkg / Feature / VTd / IntelVTdDxe / TranslationTable.c

/** @file\r
\r
  Copyright (c) 2017 - 2018, Intel Corporation. All rights reserved.<BR>\r
  This program and the accompanying materials\r
  are licensed and made available under the terms and conditions of the BSD License\r
  which accompanies this distribution.  The full text of the license may be found at\r
  http://opensource.org/licenses/bsd-license.php.\r
\r
  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
\r
**/\r
\r
#include "DmaProtection.h"\r
\r
/**\r
  Create extended context entry.\r
\r
  @param[in]  VtdIndex  The index of the VTd engine.\r
\r
  @retval EFI_SUCCESS          The extended context entry is created.\r
  @retval EFI_OUT_OF_RESOURCE  No enough resource to create extended context entry.\r
**/\r
EFI_STATUS\r
CreateExtContextEntry (\r
  IN UINTN  VtdIndex\r
  );\r
\r
/**\r
  Allocate zero pages.\r
\r
  @param[in]  Pages the number of pages.\r
\r
  @return the page address.\r
  @retval NULL No resource to allocate pages.\r
**/\r
VOID *\r
EFIAPI\r
AllocateZeroPages (\r
  IN UINTN  Pages\r
  )\r
{\r
  VOID *Addr;\r
\r
  Addr = AllocatePages (Pages);\r
  if (Addr == NULL) {\r
    return NULL;\r
  }\r
  ZeroMem (Addr, EFI_PAGES_TO_SIZE(Pages));\r
  return Addr;\r
}\r
\r
/**\r
  Set second level paging entry attribute based upon IoMmuAccess.\r
\r
  @param[in]  PtEntry      The paging entry.\r
  @param[in]  IoMmuAccess  The IOMMU access.\r
**/\r
VOID\r
SetSecondLevelPagingEntryAttribute (\r
  IN VTD_SECOND_LEVEL_PAGING_ENTRY  *PtEntry,\r
  IN UINT64                         IoMmuAccess\r
  )\r
{\r
  PtEntry->Bits.Read  = ((IoMmuAccess & EDKII_IOMMU_ACCESS_READ) != 0);\r
  PtEntry->Bits.Write = ((IoMmuAccess & EDKII_IOMMU_ACCESS_WRITE) != 0);\r
}\r
\r
/**\r
  Create context entry.\r
\r
  @param[in]  VtdIndex  The index of the VTd engine.\r
\r
  @retval EFI_SUCCESS          The context entry is created.\r
  @retval EFI_OUT_OF_RESOURCE  No enough resource to create context entry.\r
**/\r
EFI_STATUS\r
CreateContextEntry (\r
  IN UINTN  VtdIndex\r
  )\r
{\r
  UINTN                  Index;\r
  VOID                   *Buffer;\r
  UINTN                  RootPages;\r
  UINTN                  ContextPages;\r
  VTD_ROOT_ENTRY         *RootEntry;\r
  VTD_CONTEXT_ENTRY      *ContextEntryTable;\r
  VTD_CONTEXT_ENTRY      *ContextEntry;\r
  VTD_SOURCE_ID          *PciSourceId;\r
  VTD_SOURCE_ID          SourceId;\r
  UINTN                  MaxBusNumber;\r
  UINTN                  EntryTablePages;\r
\r
  MaxBusNumber = 0;\r
  for (Index = 0; Index < mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceDataNumber; Index++) {\r
    PciSourceId = &mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceData[Index].PciSourceId;\r
    if (PciSourceId->Bits.Bus > MaxBusNumber) {\r
      MaxBusNumber = PciSourceId->Bits.Bus;\r
    }\r
  }\r
  DEBUG ((DEBUG_INFO,"  MaxBusNumber - 0x%x\n", MaxBusNumber));\r
\r
  RootPages = EFI_SIZE_TO_PAGES (sizeof (VTD_ROOT_ENTRY) * VTD_ROOT_ENTRY_NUMBER);\r
  ContextPages = EFI_SIZE_TO_PAGES (sizeof (VTD_CONTEXT_ENTRY) * VTD_CONTEXT_ENTRY_NUMBER);\r
  EntryTablePages = RootPages + ContextPages * (MaxBusNumber + 1);\r
  Buffer = AllocateZeroPages (EntryTablePages);\r
  if (Buffer == NULL) {\r
    DEBUG ((DEBUG_INFO,"Could not Alloc Root Entry Table.. \n"));\r
    return EFI_OUT_OF_RESOURCES;\r
  }\r
  mVtdUnitInformation[VtdIndex].RootEntryTable = (VTD_ROOT_ENTRY *)Buffer;\r
  Buffer = (UINT8 *)Buffer + EFI_PAGES_TO_SIZE (RootPages);\r
\r
  for (Index = 0; Index < mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceDataNumber; Index++) {\r
    PciSourceId = &mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceData[Index].PciSourceId;\r
\r
    SourceId.Bits.Bus = PciSourceId->Bits.Bus;\r
    SourceId.Bits.Device = PciSourceId->Bits.Device;\r
    SourceId.Bits.Function = PciSourceId->Bits.Function;\r
\r
    RootEntry = &mVtdUnitInformation[VtdIndex].RootEntryTable[SourceId.Index.RootIndex];\r
    if (RootEntry->Bits.Present == 0) {\r
      RootEntry->Bits.ContextTablePointerLo  = (UINT32) RShiftU64 ((UINT64)(UINTN)Buffer, 12);\r
      RootEntry->Bits.ContextTablePointerHi  = (UINT32) RShiftU64 ((UINT64)(UINTN)Buffer, 32);\r
      RootEntry->Bits.Present = 1;\r
      Buffer = (UINT8 *)Buffer + EFI_PAGES_TO_SIZE (ContextPages);\r
    }\r
\r
    ContextEntryTable = (VTD_CONTEXT_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(RootEntry->Bits.ContextTablePointerLo, RootEntry->Bits.ContextTablePointerHi) ;\r
    ContextEntry = &ContextEntryTable[SourceId.Index.ContextIndex];\r
    ContextEntry->Bits.TranslationType = 0;\r
    ContextEntry->Bits.FaultProcessingDisable = 0;\r
    ContextEntry->Bits.Present = 0;\r
\r
    DEBUG ((DEBUG_INFO,"Source: S%04x B%02x D%02x F%02x\n", mVtdUnitInformation[VtdIndex].Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));\r
\r
    switch (mVtdUnitInformation[VtdIndex].CapReg.Bits.SAGAW) {\r
    case BIT1:\r
      ContextEntry->Bits.AddressWidth = 0x1;\r
      break;\r
    case BIT2:\r
      ContextEntry->Bits.AddressWidth = 0x2;\r
      break;\r
    }\r
  }\r
\r
  FlushPageTableMemory (VtdIndex, (UINTN)mVtdUnitInformation[VtdIndex].RootEntryTable, EFI_PAGES_TO_SIZE(EntryTablePages));\r
\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Create second level paging entry table.\r
\r
  @param[in]  VtdIndex                    The index of the VTd engine.\r
  @param[in]  SecondLevelPagingEntry      The second level paging entry.\r
  @param[in]  MemoryBase                  The base of the memory.\r
  @param[in]  MemoryLimit                 The limit of the memory.\r
  @param[in]  IoMmuAccess                 The IOMMU access.\r
\r
  @return The second level paging entry.\r
**/\r
VTD_SECOND_LEVEL_PAGING_ENTRY *\r
CreateSecondLevelPagingEntryTable (\r
  IN UINTN                         VtdIndex,\r
  IN VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry,\r
  IN UINT64                        MemoryBase,\r
  IN UINT64                        MemoryLimit,\r
  IN UINT64                        IoMmuAccess\r
  )\r
{\r
  UINTN                          Index4;\r
  UINTN                          Index3;\r
  UINTN                          Index2;\r
  UINTN                          Lvl4Start;\r
  UINTN                          Lvl4End;\r
  UINTN                          Lvl3Start;\r
  UINTN                          Lvl3End;\r
  VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl4PtEntry;\r
  VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl3PtEntry;\r
  VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl2PtEntry;\r
  UINT64                         BaseAddress;\r
  UINT64                         EndAddress;\r
\r
  if (MemoryLimit == 0) {\r
    return EFI_SUCCESS;\r
  }\r
\r
  BaseAddress = ALIGN_VALUE_LOW(MemoryBase, SIZE_2MB);\r
  EndAddress = ALIGN_VALUE_UP(MemoryLimit, SIZE_2MB);\r
  DEBUG ((DEBUG_INFO,"CreateSecondLevelPagingEntryTable: BaseAddress - 0x%016lx, EndAddress - 0x%016lx\n", BaseAddress, EndAddress));\r
\r
  if (SecondLevelPagingEntry == NULL) {\r
    SecondLevelPagingEntry = AllocateZeroPages (1);\r
    if (SecondLevelPagingEntry == NULL) {\r
      DEBUG ((DEBUG_ERROR,"Could not Alloc LVL4 PT. \n"));\r
      return NULL;\r
    }\r
    FlushPageTableMemory (VtdIndex, (UINTN)SecondLevelPagingEntry, EFI_PAGES_TO_SIZE(1));\r
  }\r
\r
  //\r
  // If no access is needed, just create not present entry.\r
  //\r
  if (IoMmuAccess == 0) {\r
    return SecondLevelPagingEntry;\r
  }\r
\r
  Lvl4Start = RShiftU64 (BaseAddress, 39) & 0x1FF;\r
  Lvl4End = RShiftU64 (EndAddress - 1, 39) & 0x1FF;\r
\r
  DEBUG ((DEBUG_INFO,"  Lvl4Start - 0x%x, Lvl4End - 0x%x\n", Lvl4Start, Lvl4End));\r
\r
  Lvl4PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)SecondLevelPagingEntry;\r
  for (Index4 = Lvl4Start; Index4 <= Lvl4End; Index4++) {\r
    if (Lvl4PtEntry[Index4].Uint64 == 0) {\r
      Lvl4PtEntry[Index4].Uint64 = (UINT64)(UINTN)AllocateZeroPages (1);\r
      if (Lvl4PtEntry[Index4].Uint64 == 0) {\r
        DEBUG ((DEBUG_ERROR,"!!!!!! ALLOCATE LVL4 PAGE FAIL (0x%x)!!!!!!\n", Index4));\r
        ASSERT(FALSE);\r
        return NULL;\r
      }\r
      FlushPageTableMemory (VtdIndex, (UINTN)Lvl4PtEntry[Index4].Uint64, SIZE_4KB);\r
      SetSecondLevelPagingEntryAttribute (&Lvl4PtEntry[Index4], EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);\r
    }\r
\r
    Lvl3Start = RShiftU64 (BaseAddress, 30) & 0x1FF;\r
    if (ALIGN_VALUE_LOW(BaseAddress + SIZE_1GB, SIZE_1GB) <= EndAddress) {\r
      Lvl3End = SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY) - 1;\r
    } else {\r
      Lvl3End = RShiftU64 (EndAddress - 1, 30) & 0x1FF;\r
    }\r
    DEBUG ((DEBUG_INFO,"  Lvl4(0x%x): Lvl3Start - 0x%x, Lvl3End - 0x%x\n", Index4, Lvl3Start, Lvl3End));\r
\r
    Lvl3PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl4PtEntry[Index4].Bits.AddressLo, Lvl4PtEntry[Index4].Bits.AddressHi);\r
    for (Index3 = Lvl3Start; Index3 <= Lvl3End; Index3++) {\r
      if (Lvl3PtEntry[Index3].Uint64 == 0) {\r
        Lvl3PtEntry[Index3].Uint64 = (UINT64)(UINTN)AllocateZeroPages (1);\r
        if (Lvl3PtEntry[Index3].Uint64 == 0) {\r
          DEBUG ((DEBUG_ERROR,"!!!!!! ALLOCATE LVL3 PAGE FAIL (0x%x, 0x%x)!!!!!!\n", Index4, Index3));\r
          ASSERT(FALSE);\r
          return NULL;\r
        }\r
        FlushPageTableMemory (VtdIndex, (UINTN)Lvl3PtEntry[Index3].Uint64, SIZE_4KB);\r
        SetSecondLevelPagingEntryAttribute (&Lvl3PtEntry[Index3], EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);\r
      }\r
\r
      Lvl2PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl3PtEntry[Index3].Bits.AddressLo, Lvl3PtEntry[Index3].Bits.AddressHi);\r
      for (Index2 = 0; Index2 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index2++) {\r
        Lvl2PtEntry[Index2].Uint64 = BaseAddress;\r
        SetSecondLevelPagingEntryAttribute (&Lvl2PtEntry[Index2], IoMmuAccess);\r
        Lvl2PtEntry[Index2].Bits.PageSize = 1;\r
        BaseAddress += SIZE_2MB;\r
        if (BaseAddress >= MemoryLimit) {\r
          break;\r
        }\r
      }\r
      FlushPageTableMemory (VtdIndex, (UINTN)Lvl2PtEntry, SIZE_4KB);\r
      if (BaseAddress >= MemoryLimit) {\r
        break;\r
      }\r
    }\r
    FlushPageTableMemory (VtdIndex, (UINTN)&Lvl3PtEntry[Lvl3Start], (UINTN)&Lvl3PtEntry[Lvl3End + 1] - (UINTN)&Lvl3PtEntry[Lvl3Start]);\r
    if (BaseAddress >= MemoryLimit) {\r
      break;\r
    }\r
  }\r
  FlushPageTableMemory (VtdIndex, (UINTN)&Lvl4PtEntry[Lvl4Start], (UINTN)&Lvl4PtEntry[Lvl4End + 1] - (UINTN)&Lvl4PtEntry[Lvl4Start]);\r
\r
  return SecondLevelPagingEntry;\r
}\r
\r
/**\r
  Create second level paging entry.\r
\r
  @param[in]  VtdIndex                    The index of the VTd engine.\r
  @param[in]  IoMmuAccess                 The IOMMU access.\r
\r
  @return The second level paging entry.\r
**/\r
VTD_SECOND_LEVEL_PAGING_ENTRY *\r
CreateSecondLevelPagingEntry (\r
  IN UINTN   VtdIndex,\r
  IN UINT64  IoMmuAccess\r
  )\r
{\r
  VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry;\r
\r
  SecondLevelPagingEntry = NULL;\r
  SecondLevelPagingEntry = CreateSecondLevelPagingEntryTable (VtdIndex, SecondLevelPagingEntry, 0, mBelow4GMemoryLimit, IoMmuAccess);\r
  if (SecondLevelPagingEntry == NULL) {\r
    return NULL;\r
  }\r
\r
  if (mAbove4GMemoryLimit != 0) {\r
    ASSERT (mAbove4GMemoryLimit > BASE_4GB);\r
    SecondLevelPagingEntry = CreateSecondLevelPagingEntryTable (VtdIndex, SecondLevelPagingEntry, SIZE_4GB, mAbove4GMemoryLimit, IoMmuAccess);\r
    if (SecondLevelPagingEntry == NULL) {\r
      return NULL;\r
    }\r
  }\r
\r
  return SecondLevelPagingEntry;\r
}\r
\r
/**\r
  Setup VTd translation table.\r
\r
  @retval EFI_SUCCESS           Setup translation table successfully.\r
  @retval EFI_OUT_OF_RESOURCE   Setup translation table fail.\r
**/\r
EFI_STATUS\r
SetupTranslationTable (\r
  VOID\r
  )\r
{\r
  EFI_STATUS      Status;\r
  UINTN           Index;\r
\r
  for (Index = 0; Index < mVtdUnitNumber; Index++) {\r
    DEBUG((DEBUG_INFO, "CreateContextEntry - %d\n", Index));\r
    if (mVtdUnitInformation[Index].ECapReg.Bits.ECS) {\r
      Status = CreateExtContextEntry (Index);\r
    } else {\r
      Status = CreateContextEntry (Index);\r
    }\r
    if (EFI_ERROR (Status)) {\r
      return Status;\r
    }\r
  }\r
\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Dump DMAR context entry table.\r
\r
  @param[in]  RootEntry DMAR root entry.\r
**/\r
VOID\r
DumpDmarContextEntryTable (\r
  IN VTD_ROOT_ENTRY *RootEntry\r
  )\r
{\r
  UINTN                 Index;\r
  UINTN                 Index2;\r
  VTD_CONTEXT_ENTRY     *ContextEntry;\r
\r
  DEBUG ((DEBUG_INFO,"=========================\n"));\r
  DEBUG ((DEBUG_INFO,"DMAR Context Entry Table:\n"));\r
\r
  DEBUG ((DEBUG_INFO,"RootEntry Address - 0x%x\n", RootEntry));\r
\r
  for (Index = 0; Index < VTD_ROOT_ENTRY_NUMBER; Index++) {\r
    if ((RootEntry[Index].Uint128.Uint64Lo != 0) || (RootEntry[Index].Uint128.Uint64Hi != 0)) {\r
      DEBUG ((DEBUG_INFO,"  RootEntry(0x%02x) B%02x - 0x%016lx %016lx\n",\r
        Index, Index, RootEntry[Index].Uint128.Uint64Hi, RootEntry[Index].Uint128.Uint64Lo));\r
    }\r
    if (RootEntry[Index].Bits.Present == 0) {\r
      continue;\r
    }\r
    ContextEntry = (VTD_CONTEXT_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(RootEntry[Index].Bits.ContextTablePointerLo, RootEntry[Index].Bits.ContextTablePointerHi);\r
    for (Index2 = 0; Index2 < VTD_CONTEXT_ENTRY_NUMBER; Index2++) {\r
      if ((ContextEntry[Index2].Uint128.Uint64Lo != 0) || (ContextEntry[Index2].Uint128.Uint64Hi != 0)) {\r
        DEBUG ((DEBUG_INFO,"    ContextEntry(0x%02x) D%02xF%02x - 0x%016lx %016lx\n",\r
          Index2, Index2 >> 3, Index2 & 0x7, ContextEntry[Index2].Uint128.Uint64Hi, ContextEntry[Index2].Uint128.Uint64Lo));\r
      }\r
      if (ContextEntry[Index2].Bits.Present == 0) {\r
        continue;\r
      }\r
      DumpSecondLevelPagingEntry ((VOID *)(UINTN)VTD_64BITS_ADDRESS(ContextEntry[Index2].Bits.SecondLevelPageTranslationPointerLo, ContextEntry[Index2].Bits.SecondLevelPageTranslationPointerHi));\r
    }\r
  }\r
  DEBUG ((DEBUG_INFO,"=========================\n"));\r
}\r
\r
/**\r
  Dump DMAR second level paging entry.\r
\r
  @param[in]  SecondLevelPagingEntry The second level paging entry.\r
**/\r
VOID\r
DumpSecondLevelPagingEntry (\r
  IN VOID *SecondLevelPagingEntry\r
  )\r
{\r
  UINTN                          Index4;\r
  UINTN                          Index3;\r
  UINTN                          Index2;\r
  UINTN                          Index1;\r
  VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl4PtEntry;\r
  VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl3PtEntry;\r
  VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl2PtEntry;\r
  VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl1PtEntry;\r
\r
  DEBUG ((DEBUG_VERBOSE,"================\n"));\r
  DEBUG ((DEBUG_VERBOSE,"DMAR Second Level Page Table:\n"));\r
\r
  DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry Base - 0x%x\n", SecondLevelPagingEntry));\r
  Lvl4PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)SecondLevelPagingEntry;\r
  for (Index4 = 0; Index4 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index4++) {\r
    if (Lvl4PtEntry[Index4].Uint64 != 0) {\r
      DEBUG ((DEBUG_VERBOSE,"  Lvl4Pt Entry(0x%03x) - 0x%016lx\n", Index4, Lvl4PtEntry[Index4].Uint64));\r
    }\r
    if (Lvl4PtEntry[Index4].Uint64 == 0) {\r
      continue;\r
    }\r
    Lvl3PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl4PtEntry[Index4].Bits.AddressLo, Lvl4PtEntry[Index4].Bits.AddressHi);\r
    for (Index3 = 0; Index3 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index3++) {\r
      if (Lvl3PtEntry[Index3].Uint64 != 0) {\r
        DEBUG ((DEBUG_VERBOSE,"    Lvl3Pt Entry(0x%03x) - 0x%016lx\n", Index3, Lvl3PtEntry[Index3].Uint64));\r
      }\r
      if (Lvl3PtEntry[Index3].Uint64 == 0) {\r
        continue;\r
      }\r
\r
      Lvl2PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl3PtEntry[Index3].Bits.AddressLo, Lvl3PtEntry[Index3].Bits.AddressHi);\r
      for (Index2 = 0; Index2 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index2++) {\r
        if (Lvl2PtEntry[Index2].Uint64 != 0) {\r
          DEBUG ((DEBUG_VERBOSE,"      Lvl2Pt Entry(0x%03x) - 0x%016lx\n", Index2, Lvl2PtEntry[Index2].Uint64));\r
        }\r
        if (Lvl2PtEntry[Index2].Uint64 == 0) {\r
          continue;\r
        }\r
        if (Lvl2PtEntry[Index2].Bits.PageSize == 0) {\r
          Lvl1PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl2PtEntry[Index2].Bits.AddressLo, Lvl2PtEntry[Index2].Bits.AddressHi);\r
          for (Index1 = 0; Index1 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index1++) {\r
            if (Lvl1PtEntry[Index1].Uint64 != 0) {\r
              DEBUG ((DEBUG_VERBOSE,"        Lvl1Pt Entry(0x%03x) - 0x%016lx\n", Index1, Lvl1PtEntry[Index1].Uint64));\r
            }\r
          }\r
        }\r
      }\r
    }\r
  }\r
  DEBUG ((DEBUG_VERBOSE,"================\n"));\r
}\r
\r
/**\r
  Invalid page entry.\r
\r
  @param VtdIndex  The VTd engine index.\r
**/\r
VOID\r
InvalidatePageEntry (\r
  IN UINTN                 VtdIndex\r
  )\r
{\r
  if (mVtdUnitInformation[VtdIndex].HasDirtyContext || mVtdUnitInformation[VtdIndex].HasDirtyPages) {\r
    InvalidateVtdIOTLBGlobal (VtdIndex);\r
  }\r
  mVtdUnitInformation[VtdIndex].HasDirtyContext = FALSE;\r
  mVtdUnitInformation[VtdIndex].HasDirtyPages = FALSE;\r
}\r
\r
#define VTD_PG_R                   BIT0\r
#define VTD_PG_W                   BIT1\r
#define VTD_PG_X                   BIT2\r
#define VTD_PG_EMT                 (BIT3 | BIT4 | BIT5)\r
#define VTD_PG_TM                  (BIT62)\r
\r
#define VTD_PG_PS                  BIT7\r
\r
#define PAGE_PROGATE_BITS          (VTD_PG_TM | VTD_PG_EMT | VTD_PG_W | VTD_PG_R)\r
\r
#define PAGING_4K_MASK  0xFFF\r
#define PAGING_2M_MASK  0x1FFFFF\r
#define PAGING_1G_MASK  0x3FFFFFFF\r
\r
#define PAGING_VTD_INDEX_MASK     0x1FF\r
\r
#define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull\r
#define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull\r
#define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull\r
\r
typedef enum {\r
  PageNone,\r
  Page4K,\r
  Page2M,\r
  Page1G,\r
} PAGE_ATTRIBUTE;\r
\r
typedef struct {\r
  PAGE_ATTRIBUTE   Attribute;\r
  UINT64           Length;\r
  UINT64           AddressMask;\r
} PAGE_ATTRIBUTE_TABLE;\r
\r
PAGE_ATTRIBUTE_TABLE mPageAttributeTable[] = {\r
  {Page4K,  SIZE_4KB, PAGING_4K_ADDRESS_MASK_64},\r
  {Page2M,  SIZE_2MB, PAGING_2M_ADDRESS_MASK_64},\r
  {Page1G,  SIZE_1GB, PAGING_1G_ADDRESS_MASK_64},\r
};\r
\r
/**\r
  Return length according to page attributes.\r
\r
  @param[in]  PageAttributes   The page attribute of the page entry.\r
\r
  @return The length of page entry.\r
**/\r
UINTN\r
PageAttributeToLength (\r
  IN PAGE_ATTRIBUTE  PageAttribute\r
  )\r
{\r
  UINTN  Index;\r
  for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {\r
    if (PageAttribute == mPageAttributeTable[Index].Attribute) {\r
      return (UINTN)mPageAttributeTable[Index].Length;\r
    }\r
  }\r
  return 0;\r
}\r
\r
/**\r
  Return page table entry to match the address.\r
\r
  @param[in]   VtdIndex                 The index used to identify a VTd engine.\r
  @param[in]   SecondLevelPagingEntry   The second level paging entry in VTd table for the device.\r
  @param[in]   Address                  The address to be checked.\r
  @param[out]  PageAttributes           The page attribute of the page entry.\r
\r
  @return The page entry.\r
**/\r
VOID *\r
GetSecondLevelPageTableEntry (\r
  IN  UINTN                         VtdIndex,\r
  IN  VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry,\r
  IN  PHYSICAL_ADDRESS              Address,\r
  OUT PAGE_ATTRIBUTE                *PageAttribute\r
  )\r
{\r
  UINTN                 Index1;\r
  UINTN                 Index2;\r
  UINTN                 Index3;\r
  UINTN                 Index4;\r
  UINT64                *L1PageTable;\r
  UINT64                *L2PageTable;\r
  UINT64                *L3PageTable;\r
  UINT64                *L4PageTable;\r
\r
  Index4 = ((UINTN)RShiftU64 (Address, 39)) & PAGING_VTD_INDEX_MASK;\r
  Index3 = ((UINTN)Address >> 30) & PAGING_VTD_INDEX_MASK;\r
  Index2 = ((UINTN)Address >> 21) & PAGING_VTD_INDEX_MASK;\r
  Index1 = ((UINTN)Address >> 12) & PAGING_VTD_INDEX_MASK;\r
\r
  L4PageTable = (UINT64 *)SecondLevelPagingEntry;\r
  if (L4PageTable[Index4] == 0) {\r
    L4PageTable[Index4] = (UINT64)(UINTN)AllocateZeroPages (1);\r
    if (L4PageTable[Index4] == 0) {\r
      DEBUG ((DEBUG_ERROR,"!!!!!! ALLOCATE LVL4 PAGE FAIL (0x%x)!!!!!!\n", Index4));\r
      ASSERT(FALSE);\r
      *PageAttribute = PageNone;\r
      return NULL;\r
    }\r
    FlushPageTableMemory (VtdIndex, (UINTN)L4PageTable[Index4], SIZE_4KB);\r
    SetSecondLevelPagingEntryAttribute ((VTD_SECOND_LEVEL_PAGING_ENTRY *)&L4PageTable[Index4], EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);\r
    FlushPageTableMemory (VtdIndex, (UINTN)&L4PageTable[Index4], sizeof(L4PageTable[Index4]));\r
  }\r
\r
  L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & PAGING_4K_ADDRESS_MASK_64);\r
  if (L3PageTable[Index3] == 0) {\r
    L3PageTable[Index3] = (UINT64)(UINTN)AllocateZeroPages (1);\r
    if (L3PageTable[Index3] == 0) {\r
      DEBUG ((DEBUG_ERROR,"!!!!!! ALLOCATE LVL3 PAGE FAIL (0x%x, 0x%x)!!!!!!\n", Index4, Index3));\r
      ASSERT(FALSE);\r
      *PageAttribute = PageNone;\r
      return NULL;\r
    }\r
    FlushPageTableMemory (VtdIndex, (UINTN)L3PageTable[Index3], SIZE_4KB);\r
    SetSecondLevelPagingEntryAttribute ((VTD_SECOND_LEVEL_PAGING_ENTRY *)&L3PageTable[Index3], EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);\r
    FlushPageTableMemory (VtdIndex, (UINTN)&L3PageTable[Index3], sizeof(L3PageTable[Index3]));\r
  }\r
  if ((L3PageTable[Index3] & VTD_PG_PS) != 0) {\r
    // 1G\r
    *PageAttribute = Page1G;\r
    return &L3PageTable[Index3];\r
  }\r
\r
  L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & PAGING_4K_ADDRESS_MASK_64);\r
  if (L2PageTable[Index2] == 0) {\r
    L2PageTable[Index2] = Address & PAGING_2M_ADDRESS_MASK_64;\r
    SetSecondLevelPagingEntryAttribute ((VTD_SECOND_LEVEL_PAGING_ENTRY *)&L2PageTable[Index2], 0);\r
    L2PageTable[Index2] |= VTD_PG_PS;\r
    FlushPageTableMemory (VtdIndex, (UINTN)&L2PageTable[Index2], sizeof(L2PageTable[Index2]));\r
  }\r
  if ((L2PageTable[Index2] & VTD_PG_PS) != 0) {\r
    // 2M\r
    *PageAttribute = Page2M;\r
    return &L2PageTable[Index2];\r
  }\r
\r
  // 4k\r
  L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & PAGING_4K_ADDRESS_MASK_64);\r
  if ((L1PageTable[Index1] == 0) && (Address != 0)) {\r
    *PageAttribute = PageNone;\r
    return NULL;\r
  }\r
  *PageAttribute = Page4K;\r
  return &L1PageTable[Index1];\r
}\r
\r
/**\r
  Modify memory attributes of page entry.\r
\r
  @param[in]   VtdIndex         The index used to identify a VTd engine.\r
  @param[in]   PageEntry        The page entry.\r
  @param[in]   IoMmuAccess      The IOMMU access.\r
  @param[out]  IsModified       TRUE means page table modified. FALSE means page table not modified.\r
**/\r
VOID\r
ConvertSecondLevelPageEntryAttribute (\r
  IN  UINTN                             VtdIndex,\r
  IN  VTD_SECOND_LEVEL_PAGING_ENTRY     *PageEntry,\r
  IN  UINT64                            IoMmuAccess,\r
  OUT BOOLEAN                           *IsModified\r
  )\r
{\r
  UINT64  CurrentPageEntry;\r
  UINT64  NewPageEntry;\r
\r
  CurrentPageEntry = PageEntry->Uint64;\r
  SetSecondLevelPagingEntryAttribute (PageEntry, IoMmuAccess);\r
  FlushPageTableMemory (VtdIndex, (UINTN)PageEntry, sizeof(*PageEntry));\r
  NewPageEntry = PageEntry->Uint64;\r
  if (CurrentPageEntry != NewPageEntry) {\r
    *IsModified = TRUE;\r
    DEBUG ((DEBUG_VERBOSE, "ConvertSecondLevelPageEntryAttribute 0x%lx", CurrentPageEntry));\r
    DEBUG ((DEBUG_VERBOSE, "->0x%lx\n", NewPageEntry));\r
  } else {\r
    *IsModified = FALSE;\r
  }\r
}\r
\r
/**\r
  This function returns if there is need to split page entry.\r
\r
  @param[in]  BaseAddress      The base address to be checked.\r
  @param[in]  Length           The length to be checked.\r
  @param[in]  PageAttribute    The page attribute of the page entry.\r
\r
  @retval SplitAttributes on if there is need to split page entry.\r
**/\r
PAGE_ATTRIBUTE\r
NeedSplitPage (\r
  IN  PHYSICAL_ADDRESS                  BaseAddress,\r
  IN  UINT64                            Length,\r
  IN  PAGE_ATTRIBUTE                    PageAttribute\r
  )\r
{\r
  UINT64                PageEntryLength;\r
\r
  PageEntryLength = PageAttributeToLength (PageAttribute);\r
\r
  if (((BaseAddress & (PageEntryLength - 1)) == 0) && (Length >= PageEntryLength)) {\r
    return PageNone;\r
  }\r
\r
  if (((BaseAddress & PAGING_2M_MASK) != 0) || (Length < SIZE_2MB)) {\r
    return Page4K;\r
  }\r
\r
  return Page2M;\r
}\r
\r
/**\r
  This function splits one page entry to small page entries.\r
\r
  @param[in]  VtdIndex         The index used to identify a VTd engine.\r
  @param[in]  PageEntry        The page entry to be splitted.\r
  @param[in]  PageAttribute    The page attribute of the page entry.\r
  @param[in]  SplitAttribute   How to split the page entry.\r
\r
  @retval RETURN_SUCCESS            The page entry is splitted.\r
  @retval RETURN_UNSUPPORTED        The page entry does not support to be splitted.\r
  @retval RETURN_OUT_OF_RESOURCES   No resource to split page entry.\r
**/\r
RETURN_STATUS\r
SplitSecondLevelPage (\r
  IN  UINTN                             VtdIndex,\r
  IN  VTD_SECOND_LEVEL_PAGING_ENTRY     *PageEntry,\r
  IN  PAGE_ATTRIBUTE                    PageAttribute,\r
  IN  PAGE_ATTRIBUTE                    SplitAttribute\r
  )\r
{\r
  UINT64   BaseAddress;\r
  UINT64   *NewPageEntry;\r
  UINTN    Index;\r
\r
  ASSERT (PageAttribute == Page2M || PageAttribute == Page1G);\r
\r
  if (PageAttribute == Page2M) {\r
    //\r
    // Split 2M to 4K\r
    //\r
    ASSERT (SplitAttribute == Page4K);\r
    if (SplitAttribute == Page4K) {\r
      NewPageEntry = AllocateZeroPages (1);\r
      DEBUG ((DEBUG_VERBOSE, "Split - 0x%x\n", NewPageEntry));\r
      if (NewPageEntry == NULL) {\r
        return RETURN_OUT_OF_RESOURCES;\r
      }\r
      BaseAddress = PageEntry->Uint64 & PAGING_2M_ADDRESS_MASK_64;\r
      for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {\r
        NewPageEntry[Index] = (BaseAddress + SIZE_4KB * Index) | (PageEntry->Uint64 & PAGE_PROGATE_BITS);\r
      }\r
      FlushPageTableMemory (VtdIndex, (UINTN)NewPageEntry, SIZE_4KB);\r
\r
      PageEntry->Uint64 = (UINT64)(UINTN)NewPageEntry;\r
      SetSecondLevelPagingEntryAttribute (PageEntry, EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);\r
      FlushPageTableMemory (VtdIndex, (UINTN)PageEntry, sizeof(*PageEntry));\r
      return RETURN_SUCCESS;\r
    } else {\r
      return RETURN_UNSUPPORTED;\r
    }\r
  } else if (PageAttribute == Page1G) {\r
    //\r
    // Split 1G to 2M\r
    // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.\r
    //\r
    ASSERT (SplitAttribute == Page2M || SplitAttribute == Page4K);\r
    if ((SplitAttribute == Page2M || SplitAttribute == Page4K)) {\r
      NewPageEntry = AllocateZeroPages (1);\r
      DEBUG ((DEBUG_VERBOSE, "Split - 0x%x\n", NewPageEntry));\r
      if (NewPageEntry == NULL) {\r
        return RETURN_OUT_OF_RESOURCES;\r
      }\r
      BaseAddress = PageEntry->Uint64 & PAGING_1G_ADDRESS_MASK_64;\r
      for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {\r
        NewPageEntry[Index] = (BaseAddress + SIZE_2MB * Index) | VTD_PG_PS | (PageEntry->Uint64 & PAGE_PROGATE_BITS);\r
      }\r
      FlushPageTableMemory (VtdIndex, (UINTN)NewPageEntry, SIZE_4KB);\r
\r
      PageEntry->Uint64 = (UINT64)(UINTN)NewPageEntry;\r
      SetSecondLevelPagingEntryAttribute (PageEntry, EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);\r
      FlushPageTableMemory (VtdIndex, (UINTN)PageEntry, sizeof(*PageEntry));\r
      return RETURN_SUCCESS;\r
    } else {\r
      return RETURN_UNSUPPORTED;\r
    }\r
  } else {\r
    return RETURN_UNSUPPORTED;\r
  }\r
}\r
\r
/**\r
  Set VTd attribute for a system memory on second level page entry\r
\r
  @param[in]  VtdIndex                The index used to identify a VTd engine.\r
  @param[in]  DomainIdentifier        The domain ID of the source.\r
  @param[in]  SecondLevelPagingEntry  The second level paging entry in VTd table for the device.\r
  @param[in]  BaseAddress             The base of device memory address to be used as the DMA memory.\r
  @param[in]  Length                  The length of device memory address to be used as the DMA memory.\r
  @param[in]  IoMmuAccess             The IOMMU access.\r
\r
  @retval EFI_SUCCESS            The IoMmuAccess is set for the memory range specified by BaseAddress and Length.\r
  @retval EFI_INVALID_PARAMETER  BaseAddress is not IoMmu Page size aligned.\r
  @retval EFI_INVALID_PARAMETER  Length is not IoMmu Page size aligned.\r
  @retval EFI_INVALID_PARAMETER  Length is 0.\r
  @retval EFI_INVALID_PARAMETER  IoMmuAccess specified an illegal combination of access.\r
  @retval EFI_UNSUPPORTED        The bit mask of IoMmuAccess is not supported by the IOMMU.\r
  @retval EFI_UNSUPPORTED        The IOMMU does not support the memory range specified by BaseAddress and Length.\r
  @retval EFI_OUT_OF_RESOURCES   There are not enough resources available to modify the IOMMU access.\r
  @retval EFI_DEVICE_ERROR       The IOMMU device reported an error while attempting the operation.\r
**/\r
EFI_STATUS\r
SetSecondLevelPagingAttribute (\r
  IN UINTN                         VtdIndex,\r
  IN UINT16                        DomainIdentifier,\r
  IN VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry,\r
  IN UINT64                        BaseAddress,\r
  IN UINT64                        Length,\r
  IN UINT64                        IoMmuAccess\r
  )\r
{\r
  VTD_SECOND_LEVEL_PAGING_ENTRY  *PageEntry;\r
  PAGE_ATTRIBUTE                 PageAttribute;\r
  UINTN                          PageEntryLength;\r
  PAGE_ATTRIBUTE                 SplitAttribute;\r
  EFI_STATUS                     Status;\r
  BOOLEAN                        IsEntryModified;\r
\r
  DEBUG ((DEBUG_VERBOSE,"SetSecondLevelPagingAttribute (%d) (0x%016lx - 0x%016lx : %x) \n", VtdIndex, BaseAddress, Length, IoMmuAccess));\r
  DEBUG ((DEBUG_VERBOSE,"  SecondLevelPagingEntry Base - 0x%x\n", SecondLevelPagingEntry));\r
\r
  if (BaseAddress != ALIGN_VALUE(BaseAddress, SIZE_4KB)) {\r
    DEBUG ((DEBUG_ERROR, "SetSecondLevelPagingAttribute - Invalid Alignment\n"));\r
    return EFI_UNSUPPORTED;\r
  }\r
  if (Length != ALIGN_VALUE(Length, SIZE_4KB)) {\r
    DEBUG ((DEBUG_ERROR, "SetSecondLevelPagingAttribute - Invalid Alignment\n"));\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  while (Length != 0) {\r
    PageEntry = GetSecondLevelPageTableEntry (VtdIndex, SecondLevelPagingEntry, BaseAddress, &PageAttribute);\r
    if (PageEntry == NULL) {\r
      DEBUG ((DEBUG_ERROR, "PageEntry - NULL\n"));\r
      return RETURN_UNSUPPORTED;\r
    }\r
    PageEntryLength = PageAttributeToLength (PageAttribute);\r
    SplitAttribute = NeedSplitPage (BaseAddress, Length, PageAttribute);\r
    if (SplitAttribute == PageNone) {\r
      ConvertSecondLevelPageEntryAttribute (VtdIndex, PageEntry, IoMmuAccess, &IsEntryModified);\r
      if (IsEntryModified) {\r
        mVtdUnitInformation[VtdIndex].HasDirtyPages = TRUE;\r
      }\r
      //\r
      // Convert success, move to next\r
      //\r
      BaseAddress += PageEntryLength;\r
      Length -= PageEntryLength;\r
    } else {\r
      Status = SplitSecondLevelPage (VtdIndex, PageEntry, PageAttribute, SplitAttribute);\r
      if (RETURN_ERROR (Status)) {\r
        DEBUG ((DEBUG_ERROR, "SplitSecondLevelPage - %r\n", Status));\r
        return RETURN_UNSUPPORTED;\r
      }\r
      mVtdUnitInformation[VtdIndex].HasDirtyPages = TRUE;\r
      //\r
      // Just split current page\r
      // Convert success in next around\r
      //\r
    }\r
  }\r
\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Set VTd attribute for a system memory.\r
\r
  @param[in]  VtdIndex                The index used to identify a VTd engine.\r
  @param[in]  DomainIdentifier        The domain ID of the source.\r
  @param[in]  SecondLevelPagingEntry  The second level paging entry in VTd table for the device.\r
  @param[in]  BaseAddress             The base of device memory address to be used as the DMA memory.\r
  @param[in]  Length                  The length of device memory address to be used as the DMA memory.\r
  @param[in]  IoMmuAccess             The IOMMU access.\r
\r
  @retval EFI_SUCCESS            The IoMmuAccess is set for the memory range specified by BaseAddress and Length.\r
  @retval EFI_INVALID_PARAMETER  BaseAddress is not IoMmu Page size aligned.\r
  @retval EFI_INVALID_PARAMETER  Length is not IoMmu Page size aligned.\r
  @retval EFI_INVALID_PARAMETER  Length is 0.\r
  @retval EFI_INVALID_PARAMETER  IoMmuAccess specified an illegal combination of access.\r
  @retval EFI_UNSUPPORTED        The bit mask of IoMmuAccess is not supported by the IOMMU.\r
  @retval EFI_UNSUPPORTED        The IOMMU does not support the memory range specified by BaseAddress and Length.\r
  @retval EFI_OUT_OF_RESOURCES   There are not enough resources available to modify the IOMMU access.\r
  @retval EFI_DEVICE_ERROR       The IOMMU device reported an error while attempting the operation.\r
**/\r
EFI_STATUS\r
SetPageAttribute (\r
  IN UINTN                         VtdIndex,\r
  IN UINT16                        DomainIdentifier,\r
  IN VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry,\r
  IN UINT64                        BaseAddress,\r
  IN UINT64                        Length,\r
  IN UINT64                        IoMmuAccess\r
  )\r
{\r
  EFI_STATUS Status;\r
  Status = EFI_NOT_FOUND;\r
  if (SecondLevelPagingEntry != NULL) {\r
    Status = SetSecondLevelPagingAttribute (VtdIndex, DomainIdentifier, SecondLevelPagingEntry, BaseAddress, Length, IoMmuAccess);\r
  }\r
  return Status;\r
}\r
\r
/**\r
  Set VTd attribute for a system memory.\r
\r
  @param[in]  Segment           The Segment used to identify a VTd engine.\r
  @param[in]  SourceId          The SourceId used to identify a VTd engine and table entry.\r
  @param[in]  BaseAddress       The base of device memory address to be used as the DMA memory.\r
  @param[in]  Length            The length of device memory address to be used as the DMA memory.\r
  @param[in]  IoMmuAccess       The IOMMU access.\r
\r
  @retval EFI_SUCCESS            The IoMmuAccess is set for the memory range specified by BaseAddress and Length.\r
  @retval EFI_INVALID_PARAMETER  BaseAddress is not IoMmu Page size aligned.\r
  @retval EFI_INVALID_PARAMETER  Length is not IoMmu Page size aligned.\r
  @retval EFI_INVALID_PARAMETER  Length is 0.\r
  @retval EFI_INVALID_PARAMETER  IoMmuAccess specified an illegal combination of access.\r
  @retval EFI_UNSUPPORTED        The bit mask of IoMmuAccess is not supported by the IOMMU.\r
  @retval EFI_UNSUPPORTED        The IOMMU does not support the memory range specified by BaseAddress and Length.\r
  @retval EFI_OUT_OF_RESOURCES   There are not enough resources available to modify the IOMMU access.\r
  @retval EFI_DEVICE_ERROR       The IOMMU device reported an error while attempting the operation.\r
**/\r
EFI_STATUS\r
SetAccessAttribute (\r
  IN UINT16                Segment,\r
  IN VTD_SOURCE_ID         SourceId,\r
  IN UINT64                BaseAddress,\r
  IN UINT64                Length,\r
  IN UINT64                IoMmuAccess\r
  )\r
{\r
  UINTN                         VtdIndex;\r
  EFI_STATUS                    Status;\r
  VTD_EXT_CONTEXT_ENTRY         *ExtContextEntry;\r
  VTD_CONTEXT_ENTRY             *ContextEntry;\r
  VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry;\r
  UINT64                        Pt;\r
  UINTN                         PciDataIndex;\r
  UINT16                        DomainIdentifier;\r
\r
  SecondLevelPagingEntry = NULL;\r
\r
  DEBUG ((DEBUG_VERBOSE,"SetAccessAttribute (S%04x B%02x D%02x F%02x) (0x%016lx - 0x%08x, %x)\n", Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function, BaseAddress, (UINTN)Length, IoMmuAccess));\r
\r
  VtdIndex = FindVtdIndexByPciDevice (Segment, SourceId, &ExtContextEntry, &ContextEntry);\r
  if (VtdIndex == (UINTN)-1) {\r
    DEBUG ((DEBUG_ERROR,"SetAccessAttribute - Pci device (S%04x B%02x D%02x F%02x) not found!\n", Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));\r
    return EFI_DEVICE_ERROR;\r
  }\r
\r
  PciDataIndex = GetPciDataIndex (VtdIndex, Segment, SourceId);\r
  mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceData[PciDataIndex].AccessCount++;\r
  //\r
  // DomainId should not be 0.\r
  //\r
  DomainIdentifier = (UINT16)(PciDataIndex + 1);\r
\r
  if (ExtContextEntry != NULL) {\r
    if (ExtContextEntry->Bits.Present == 0) {\r
      SecondLevelPagingEntry = CreateSecondLevelPagingEntry (VtdIndex, 0);\r
      DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry - 0x%x (S%04x B%02x D%02x F%02x) New\n", SecondLevelPagingEntry, Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));\r
      Pt = (UINT64)RShiftU64 ((UINT64)(UINTN)SecondLevelPagingEntry, 12);\r
\r
      ExtContextEntry->Bits.SecondLevelPageTranslationPointerLo = (UINT32) Pt;\r
      ExtContextEntry->Bits.SecondLevelPageTranslationPointerHi = (UINT32) RShiftU64(Pt, 20);\r
      ExtContextEntry->Bits.DomainIdentifier = DomainIdentifier;\r
      ExtContextEntry->Bits.Present = 1;\r
      FlushPageTableMemory (VtdIndex, (UINTN)ExtContextEntry, sizeof(*ExtContextEntry));\r
      DumpDmarExtContextEntryTable (mVtdUnitInformation[VtdIndex].ExtRootEntryTable);\r
      mVtdUnitInformation[VtdIndex].HasDirtyContext = TRUE;\r
    } else {\r
      SecondLevelPagingEntry = (VOID *)(UINTN)VTD_64BITS_ADDRESS(ExtContextEntry->Bits.SecondLevelPageTranslationPointerLo, ExtContextEntry->Bits.SecondLevelPageTranslationPointerHi);\r
      DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry - 0x%x (S%04x B%02x D%02x F%02x)\n", SecondLevelPagingEntry, Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));\r
    }\r
  } else if (ContextEntry != NULL) {\r
    if (ContextEntry->Bits.Present == 0) {\r
      SecondLevelPagingEntry = CreateSecondLevelPagingEntry (VtdIndex, 0);\r
      DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry - 0x%x (S%04x B%02x D%02x F%02x) New\n", SecondLevelPagingEntry, Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));\r
      Pt = (UINT64)RShiftU64 ((UINT64)(UINTN)SecondLevelPagingEntry, 12);\r
\r
      ContextEntry->Bits.SecondLevelPageTranslationPointerLo = (UINT32) Pt;\r
      ContextEntry->Bits.SecondLevelPageTranslationPointerHi = (UINT32) RShiftU64(Pt, 20);\r
      ContextEntry->Bits.DomainIdentifier = DomainIdentifier;\r
      ContextEntry->Bits.Present = 1;\r
      FlushPageTableMemory (VtdIndex, (UINTN)ContextEntry, sizeof(*ContextEntry));\r
      DumpDmarContextEntryTable (mVtdUnitInformation[VtdIndex].RootEntryTable);\r
      mVtdUnitInformation[VtdIndex].HasDirtyContext = TRUE;\r
    } else {\r
      SecondLevelPagingEntry = (VOID *)(UINTN)VTD_64BITS_ADDRESS(ContextEntry->Bits.SecondLevelPageTranslationPointerLo, ContextEntry->Bits.SecondLevelPageTranslationPointerHi);\r
      DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry - 0x%x (S%04x B%02x D%02x F%02x)\n", SecondLevelPagingEntry, Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));\r
    }\r
  }\r
\r
  //\r
  // Do not update FixedSecondLevelPagingEntry\r
  //\r
  if (SecondLevelPagingEntry != mVtdUnitInformation[VtdIndex].FixedSecondLevelPagingEntry) {\r
    Status = SetPageAttribute (\r
               VtdIndex,\r
               DomainIdentifier,\r
               SecondLevelPagingEntry,\r
               BaseAddress,\r
               Length,\r
               IoMmuAccess\r
               );\r
    if (EFI_ERROR (Status)) {\r
      DEBUG ((DEBUG_ERROR,"SetPageAttribute - %r\n", Status));\r
      return Status;\r
    }\r
  }\r
\r
  InvalidatePageEntry (VtdIndex);\r
\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Always enable the VTd page attribute for the device.\r
\r
  @param[in]  Segment           The Segment used to identify a VTd engine.\r
  @param[in]  SourceId          The SourceId used to identify a VTd engine and table entry.\r
\r
  @retval EFI_SUCCESS           The VTd entry is updated to always enable all DMA access for the specific device.\r
**/\r
EFI_STATUS\r
AlwaysEnablePageAttribute (\r
  IN UINT16                  Segment,\r
  IN VTD_SOURCE_ID           SourceId\r
  )\r
{\r
  UINTN                         VtdIndex;\r
  VTD_EXT_CONTEXT_ENTRY         *ExtContextEntry;\r
  VTD_CONTEXT_ENTRY             *ContextEntry;\r
  VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry;\r
  UINT64                        Pt;\r
\r
  DEBUG ((DEBUG_INFO,"AlwaysEnablePageAttribute (S%04x B%02x D%02x F%02x)\n", Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));\r
\r
  VtdIndex = FindVtdIndexByPciDevice (Segment, SourceId, &ExtContextEntry, &ContextEntry);\r
  if (VtdIndex == (UINTN)-1) {\r
    DEBUG ((DEBUG_ERROR,"AlwaysEnablePageAttribute - Pci device (S%04x B%02x D%02x F%02x) not found!\n", Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));\r
    return EFI_DEVICE_ERROR;\r
  }\r
\r
  if (mVtdUnitInformation[VtdIndex].FixedSecondLevelPagingEntry == 0) {\r
    DEBUG((DEBUG_INFO, "CreateSecondLevelPagingEntry - %d\n", VtdIndex));\r
    mVtdUnitInformation[VtdIndex].FixedSecondLevelPagingEntry = CreateSecondLevelPagingEntry (VtdIndex, EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);\r
  }\r
\r
  SecondLevelPagingEntry = mVtdUnitInformation[VtdIndex].FixedSecondLevelPagingEntry;\r
  Pt = (UINT64)RShiftU64 ((UINT64)(UINTN)SecondLevelPagingEntry, 12);\r
  if (ExtContextEntry != NULL) {\r
    ExtContextEntry->Bits.SecondLevelPageTranslationPointerLo = (UINT32) Pt;\r
    ExtContextEntry->Bits.SecondLevelPageTranslationPointerHi = (UINT32) RShiftU64(Pt, 20);\r
    ExtContextEntry->Bits.DomainIdentifier = ((1 << (UINT8)((UINTN)mVtdUnitInformation[VtdIndex].CapReg.Bits.ND * 2 + 4)) - 1);\r
    ExtContextEntry->Bits.Present = 1;\r
    FlushPageTableMemory (VtdIndex, (UINTN)ExtContextEntry, sizeof(*ExtContextEntry));\r
  } else if (ContextEntry != NULL) {\r
    ContextEntry->Bits.SecondLevelPageTranslationPointerLo = (UINT32) Pt;\r
    ContextEntry->Bits.SecondLevelPageTranslationPointerHi = (UINT32) RShiftU64(Pt, 20);\r
    ContextEntry->Bits.DomainIdentifier = ((1 << (UINT8)((UINTN)mVtdUnitInformation[VtdIndex].CapReg.Bits.ND * 2 + 4)) - 1);\r
    ContextEntry->Bits.Present = 1;\r
    FlushPageTableMemory (VtdIndex, (UINTN)ContextEntry, sizeof(*ContextEntry));\r
  }\r
\r
  return EFI_SUCCESS;\r
}\r