IntelSiliconPkg/Feature/VTd/IntelVTdDxe/TranslationTable.c

   1 /** @file
   2
   3   Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
   4   This program and the accompanying materials
   5   are licensed and made available under the terms and conditions of the BSD License
   6   which accompanies this distribution.  The full text of the license may be found at
   7   http://opensource.org/licenses/bsd-license.php.
   8
   9   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  10   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  11
  12 **/
  13
  14 #include "DmaProtection.h"
  15
  16 /**
  17   Create extended context entry.
  18
  19   @param[in]  VtdIndex  The index of the VTd engine.
  20
  21   @retval EFI_SUCCESS          The extended context entry is created.
  22   @retval EFI_OUT_OF_RESOURCE  No enough resource to create extended context entry.
  23 **/
  24 EFI_STATUS
  25 CreateExtContextEntry (
  26   IN UINTN  VtdIndex
  27   );
  28
  29 /**
  30   Allocate zero pages.
  31
  32   @param[in]  Pages the number of pages.
  33
  34   @return the page address.
  35   @retval NULL No resource to allocate pages.
  36 **/
  37 VOID *
  38 EFIAPI
  39 AllocateZeroPages (
  40   IN UINTN  Pages
  41   )
  42 {
  43   VOID *Addr;
  44
  45   Addr = AllocatePages (Pages);
  46   if (Addr == NULL) {
  47     return NULL;
  48   }
  49   ZeroMem (Addr, EFI_PAGES_TO_SIZE(Pages));
  50   return Addr;
  51 }
  52
  53 /**
  54   Set second level paging entry attribute based upon IoMmuAccess.
  55
  56   @param[in]  PtEntry      The paging entry.
  57   @param[in]  IoMmuAccess  The IOMMU access.
  58 **/
  59 VOID
  60 SetSecondLevelPagingEntryAttribute (
  61   IN VTD_SECOND_LEVEL_PAGING_ENTRY  *PtEntry,
  62   IN UINT64                         IoMmuAccess
  63   )
  64 {
  65   PtEntry->Bits.Read  = ((IoMmuAccess & EDKII_IOMMU_ACCESS_READ) != 0);
  66   PtEntry->Bits.Write = ((IoMmuAccess & EDKII_IOMMU_ACCESS_WRITE) != 0);
  67 }
  68
  69 /**
  70   Create context entry.
  71
  72   @param[in]  VtdIndex  The index of the VTd engine.
  73
  74   @retval EFI_SUCCESS          The context entry is created.
  75   @retval EFI_OUT_OF_RESOURCE  No enough resource to create context entry.
  76 **/
  77 EFI_STATUS
  78 CreateContextEntry (
  79   IN UINTN  VtdIndex
  80   )
  81 {
  82   UINTN                  Index;
  83   VOID                   *Buffer;
  84   UINTN                  RootPages;
  85   UINTN                  ContextPages;
  86   VTD_ROOT_ENTRY         *RootEntry;
  87   VTD_CONTEXT_ENTRY      *ContextEntryTable;
  88   VTD_CONTEXT_ENTRY      *ContextEntry;
  89   VTD_SOURCE_ID          *PciSourceId;
  90   VTD_SOURCE_ID          SourceId;
  91   UINTN                  MaxBusNumber;
  92   UINTN                  EntryTablePages;
  93
  94   MaxBusNumber = 0;
  95   for (Index = 0; Index < mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceDataNumber; Index++) {
  96     PciSourceId = &mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceData[Index].PciSourceId;
  97     if (PciSourceId->Bits.Bus > MaxBusNumber) {
  98       MaxBusNumber = PciSourceId->Bits.Bus;
  99     }
 100   }
 101   DEBUG ((DEBUG_INFO,"  MaxBusNumber - 0x%x\n", MaxBusNumber));
 102
 103   RootPages = EFI_SIZE_TO_PAGES (sizeof (VTD_ROOT_ENTRY) * VTD_ROOT_ENTRY_NUMBER);
 104   ContextPages = EFI_SIZE_TO_PAGES (sizeof (VTD_CONTEXT_ENTRY) * VTD_CONTEXT_ENTRY_NUMBER);
 105   EntryTablePages = RootPages + ContextPages * (MaxBusNumber + 1);
 106   Buffer = AllocateZeroPages (EntryTablePages);
 107   if (Buffer == NULL) {
 108     DEBUG ((DEBUG_INFO,"Could not Alloc Root Entry Table.. \n"));
 109     return EFI_OUT_OF_RESOURCES;
 110   }
 111   mVtdUnitInformation[VtdIndex].RootEntryTable = (VTD_ROOT_ENTRY *)Buffer;
 112   Buffer = (UINT8 *)Buffer + EFI_PAGES_TO_SIZE (RootPages);
 113
 114   for (Index = 0; Index < mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceDataNumber; Index++) {
 115     PciSourceId = &mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceData[Index].PciSourceId;
 116
 117     SourceId.Bits.Bus = PciSourceId->Bits.Bus;
 118     SourceId.Bits.Device = PciSourceId->Bits.Device;
 119     SourceId.Bits.Function = PciSourceId->Bits.Function;
 120
 121     RootEntry = &mVtdUnitInformation[VtdIndex].RootEntryTable[SourceId.Index.RootIndex];
 122     if (RootEntry->Bits.Present == 0) {
 123       RootEntry->Bits.ContextTablePointerLo  = (UINT32) RShiftU64 ((UINT64)(UINTN)Buffer, 12);
 124       RootEntry->Bits.ContextTablePointerHi  = (UINT32) RShiftU64 ((UINT64)(UINTN)Buffer, 32);
 125       RootEntry->Bits.Present = 1;
 126       Buffer = (UINT8 *)Buffer + EFI_PAGES_TO_SIZE (ContextPages);
 127     }
 128
 129     ContextEntryTable = (VTD_CONTEXT_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(RootEntry->Bits.ContextTablePointerLo, RootEntry->Bits.ContextTablePointerHi) ;
 130     ContextEntry = &ContextEntryTable[SourceId.Index.ContextIndex];
 131     ContextEntry->Bits.TranslationType = 0;
 132     ContextEntry->Bits.FaultProcessingDisable = 0;
 133     ContextEntry->Bits.Present = 0;
 134
 135     DEBUG ((DEBUG_INFO,"Source: S%04x B%02x D%02x F%02x\n", mVtdUnitInformation[VtdIndex].Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 136
 137     switch (mVtdUnitInformation[VtdIndex].CapReg.Bits.SAGAW) {
 138     case BIT1:
 139       ContextEntry->Bits.AddressWidth = 0x1;
 140       break;
 141     case BIT2:
 142       ContextEntry->Bits.AddressWidth = 0x2;
 143       break;
 144     }
 145   }
 146
 147   FlushPageTableMemory (VtdIndex, (UINTN)mVtdUnitInformation[VtdIndex].RootEntryTable, EFI_PAGES_TO_SIZE(EntryTablePages));
 148
 149   return EFI_SUCCESS;
 150 }
 151
 152 /**
 153   Create second level paging entry table.
 154
 155   @param[in]  VtdIndex                    The index of the VTd engine.
 156   @param[in]  SecondLevelPagingEntry      The second level paging entry.
 157   @param[in]  MemoryBase                  The base of the memory.
 158   @param[in]  MemoryLimit                 The limit of the memory.
 159   @param[in]  IoMmuAccess                 The IOMMU access.
 160
 161   @return The second level paging entry.
 162 **/
 163 VTD_SECOND_LEVEL_PAGING_ENTRY *
 164 CreateSecondLevelPagingEntryTable (
 165   IN UINTN                         VtdIndex,
 166   IN VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry,
 167   IN UINT64                        MemoryBase,
 168   IN UINT64                        MemoryLimit,
 169   IN UINT64                        IoMmuAccess
 170   )
 171 {
 172   UINTN                          Index4;
 173   UINTN                          Index3;
 174   UINTN                          Index2;
 175   UINTN                          Lvl4Start;
 176   UINTN                          Lvl4End;
 177   UINTN                          Lvl3Start;
 178   UINTN                          Lvl3End;
 179   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl4PtEntry;
 180   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl3PtEntry;
 181   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl2PtEntry;
 182   UINT64                         BaseAddress;
 183   UINT64                         EndAddress;
 184
 185   if (MemoryLimit == 0) {
 186     return EFI_SUCCESS;
 187   }
 188
 189   BaseAddress = ALIGN_VALUE_LOW(MemoryBase, SIZE_2MB);
 190   EndAddress = ALIGN_VALUE_UP(MemoryLimit, SIZE_2MB);
 191   DEBUG ((DEBUG_INFO,"CreateSecondLevelPagingEntryTable: BaseAddress - 0x%016lx, EndAddress - 0x%016lx\n", BaseAddress, EndAddress));
 192
 193   if (SecondLevelPagingEntry == NULL) {
 194     SecondLevelPagingEntry = AllocateZeroPages (1);
 195     if (SecondLevelPagingEntry == NULL) {
 196       DEBUG ((DEBUG_ERROR,"Could not Alloc LVL4 PT. \n"));
 197       return NULL;
 198     }
 199     FlushPageTableMemory (VtdIndex, (UINTN)SecondLevelPagingEntry, EFI_PAGES_TO_SIZE(1));
 200   }
 201
 202   //
 203   // If no access is needed, just create not present entry.
 204   //
 205   if (IoMmuAccess == 0) {
 206     return SecondLevelPagingEntry;
 207   }
 208
 209   Lvl4Start = RShiftU64 (BaseAddress, 39) & 0x1FF;
 210   Lvl4End = RShiftU64 (EndAddress - 1, 39) & 0x1FF;
 211
 212   DEBUG ((DEBUG_INFO,"  Lvl4Start - 0x%x, Lvl4End - 0x%x\n", Lvl4Start, Lvl4End));
 213
 214   Lvl4PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)SecondLevelPagingEntry;
 215   for (Index4 = Lvl4Start; Index4 <= Lvl4End; Index4++) {
 216     if (Lvl4PtEntry[Index4].Uint64 == 0) {
 217       Lvl4PtEntry[Index4].Uint64 = (UINT64)(UINTN)AllocateZeroPages (1);
 218       if (Lvl4PtEntry[Index4].Uint64 == 0) {
 219         DEBUG ((DEBUG_ERROR,"!!!!!! ALLOCATE LVL4 PAGE FAIL (0x%x)!!!!!!\n", Index4));
 220         ASSERT(FALSE);
 221         return NULL;
 222       }
 223       FlushPageTableMemory (VtdIndex, (UINTN)Lvl4PtEntry[Index4].Uint64, SIZE_4KB);
 224       SetSecondLevelPagingEntryAttribute (&Lvl4PtEntry[Index4], EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 225     }
 226
 227     Lvl3Start = RShiftU64 (BaseAddress, 30) & 0x1FF;
 228     if (ALIGN_VALUE_LOW(BaseAddress + SIZE_1GB, SIZE_1GB) <= EndAddress) {
 229       Lvl3End = SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY);
 230     } else {
 231       Lvl3End = RShiftU64 (EndAddress - 1, 30) & 0x1FF;
 232     }
 233     DEBUG ((DEBUG_INFO,"  Lvl4(0x%x): Lvl3Start - 0x%x, Lvl3End - 0x%x\n", Index4, Lvl3Start, Lvl3End));
 234
 235     Lvl3PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl4PtEntry[Index4].Bits.AddressLo, Lvl4PtEntry[Index4].Bits.AddressHi);
 236     for (Index3 = Lvl3Start; Index3 <= Lvl3End; Index3++) {
 237       if (Lvl3PtEntry[Index3].Uint64 == 0) {
 238         Lvl3PtEntry[Index3].Uint64 = (UINT64)(UINTN)AllocateZeroPages (1);
 239         if (Lvl3PtEntry[Index3].Uint64 == 0) {
 240           DEBUG ((DEBUG_ERROR,"!!!!!! ALLOCATE LVL3 PAGE FAIL (0x%x, 0x%x)!!!!!!\n", Index4, Index3));
 241           ASSERT(FALSE);
 242           return NULL;
 243         }
 244         FlushPageTableMemory (VtdIndex, (UINTN)Lvl3PtEntry[Index3].Uint64, SIZE_4KB);
 245         SetSecondLevelPagingEntryAttribute (&Lvl3PtEntry[Index3], EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 246       }
 247
 248       Lvl2PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl3PtEntry[Index3].Bits.AddressLo, Lvl3PtEntry[Index3].Bits.AddressHi);
 249       for (Index2 = 0; Index2 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index2++) {
 250         Lvl2PtEntry[Index2].Uint64 = BaseAddress;
 251         SetSecondLevelPagingEntryAttribute (&Lvl2PtEntry[Index2], IoMmuAccess);
 252         Lvl2PtEntry[Index2].Bits.PageSize = 1;
 253         BaseAddress += SIZE_2MB;
 254         if (BaseAddress >= MemoryLimit) {
 255           goto Done;
 256         }
 257       }
 258       FlushPageTableMemory (VtdIndex, (UINTN)Lvl2PtEntry, SIZE_4KB);
 259     }
 260     FlushPageTableMemory (VtdIndex, (UINTN)&Lvl3PtEntry[Lvl3Start], (UINTN)&Lvl3PtEntry[Lvl3End + 1] - (UINTN)&Lvl3PtEntry[Lvl3Start]);
 261   }
 262   FlushPageTableMemory (VtdIndex, (UINTN)&Lvl4PtEntry[Lvl4Start], (UINTN)&Lvl4PtEntry[Lvl4End + 1] - (UINTN)&Lvl4PtEntry[Lvl4Start]);
 263
 264 Done:
 265   return SecondLevelPagingEntry;
 266 }
 267
 268 /**
 269   Create second level paging entry.
 270
 271   @param[in]  VtdIndex                    The index of the VTd engine.
 272   @param[in]  IoMmuAccess                 The IOMMU access.
 273
 274   @return The second level paging entry.
 275 **/
 276 VTD_SECOND_LEVEL_PAGING_ENTRY *
 277 CreateSecondLevelPagingEntry (
 278   IN UINTN   VtdIndex,
 279   IN UINT64  IoMmuAccess
 280   )
 281 {
 282   VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry;
 283
 284   SecondLevelPagingEntry = NULL;
 285   SecondLevelPagingEntry = CreateSecondLevelPagingEntryTable (VtdIndex, SecondLevelPagingEntry, 0, mBelow4GMemoryLimit, IoMmuAccess);
 286   if (SecondLevelPagingEntry == NULL) {
 287     return NULL;
 288   }
 289   SecondLevelPagingEntry = CreateSecondLevelPagingEntryTable (VtdIndex, SecondLevelPagingEntry, SIZE_4GB, mAbove4GMemoryLimit, IoMmuAccess);
 290   if (SecondLevelPagingEntry == NULL) {
 291     return NULL;
 292   }
 293
 294   return SecondLevelPagingEntry;
 295 }
 296
 297 /**
 298   Setup VTd translation table.
 299
 300   @retval EFI_SUCCESS           Setup translation table successfully.
 301   @retval EFI_OUT_OF_RESOURCE   Setup translation table fail.
 302 **/
 303 EFI_STATUS
 304 SetupTranslationTable (
 305   VOID
 306   )
 307 {
 308   EFI_STATUS      Status;
 309   UINTN           Index;
 310
 311   for (Index = 0; Index < mVtdUnitNumber; Index++) {
 312     DEBUG((DEBUG_INFO, "CreateContextEntry - %d\n", Index));
 313     if (mVtdUnitInformation[Index].ECapReg.Bits.ECS) {
 314       Status = CreateExtContextEntry (Index);
 315     } else {
 316       Status = CreateContextEntry (Index);
 317     }
 318     if (EFI_ERROR (Status)) {
 319       return Status;
 320     }
 321   }
 322
 323   return EFI_SUCCESS;
 324 }
 325
 326 /**
 327   Dump DMAR context entry table.
 328
 329   @param[in]  RootEntry DMAR root entry.
 330 **/
 331 VOID
 332 DumpDmarContextEntryTable (
 333   IN VTD_ROOT_ENTRY *RootEntry
 334   )
 335 {
 336   UINTN                 Index;
 337   UINTN                 Index2;
 338   VTD_CONTEXT_ENTRY     *ContextEntry;
 339
 340   DEBUG ((DEBUG_INFO,"=========================\n"));
 341   DEBUG ((DEBUG_INFO,"DMAR Context Entry Table:\n"));
 342
 343   DEBUG ((DEBUG_INFO,"RootEntry Address - 0x%x\n", RootEntry));
 344
 345   for (Index = 0; Index < VTD_ROOT_ENTRY_NUMBER; Index++) {
 346     if ((RootEntry[Index].Uint128.Uint64Lo != 0) || (RootEntry[Index].Uint128.Uint64Hi != 0)) {
 347       DEBUG ((DEBUG_INFO,"  RootEntry(0x%02x) B%02x - 0x%016lx %016lx\n",
 348         Index, Index, RootEntry[Index].Uint128.Uint64Hi, RootEntry[Index].Uint128.Uint64Lo));
 349     }
 350     if (RootEntry[Index].Bits.Present == 0) {
 351       continue;
 352     }
 353     ContextEntry = (VTD_CONTEXT_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(RootEntry[Index].Bits.ContextTablePointerLo, RootEntry[Index].Bits.ContextTablePointerHi);
 354     for (Index2 = 0; Index2 < VTD_CONTEXT_ENTRY_NUMBER; Index2++) {
 355       if ((ContextEntry[Index2].Uint128.Uint64Lo != 0) || (ContextEntry[Index2].Uint128.Uint64Hi != 0)) {
 356         DEBUG ((DEBUG_INFO,"    ContextEntry(0x%02x) D%02xF%02x - 0x%016lx %016lx\n",
 357           Index2, Index2 >> 3, Index2 & 0x7, ContextEntry[Index2].Uint128.Uint64Hi, ContextEntry[Index2].Uint128.Uint64Lo));
 358       }
 359       if (ContextEntry[Index2].Bits.Present == 0) {
 360         continue;
 361       }
 362       DumpSecondLevelPagingEntry ((VOID *)(UINTN)VTD_64BITS_ADDRESS(ContextEntry[Index2].Bits.SecondLevelPageTranslationPointerLo, ContextEntry[Index2].Bits.SecondLevelPageTranslationPointerHi));
 363     }
 364   }
 365   DEBUG ((DEBUG_INFO,"=========================\n"));
 366 }
 367
 368 /**
 369   Dump DMAR second level paging entry.
 370
 371   @param[in]  SecondLevelPagingEntry The second level paging entry.
 372 **/
 373 VOID
 374 DumpSecondLevelPagingEntry (
 375   IN VOID *SecondLevelPagingEntry
 376   )
 377 {
 378   UINTN                          Index4;
 379   UINTN                          Index3;
 380   UINTN                          Index2;
 381   UINTN                          Index1;
 382   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl4PtEntry;
 383   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl3PtEntry;
 384   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl2PtEntry;
 385   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl1PtEntry;
 386
 387   DEBUG ((DEBUG_VERBOSE,"================\n"));
 388   DEBUG ((DEBUG_VERBOSE,"DMAR Second Level Page Table:\n"));
 389
 390   DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry Base - 0x%x\n", SecondLevelPagingEntry));
 391   Lvl4PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)SecondLevelPagingEntry;
 392   for (Index4 = 0; Index4 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index4++) {
 393     if (Lvl4PtEntry[Index4].Uint64 != 0) {
 394       DEBUG ((DEBUG_VERBOSE,"  Lvl4Pt Entry(0x%03x) - 0x%016lx\n", Index4, Lvl4PtEntry[Index4].Uint64));
 395     }
 396     if (Lvl4PtEntry[Index4].Uint64 == 0) {
 397       continue;
 398     }
 399     Lvl3PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl4PtEntry[Index4].Bits.AddressLo, Lvl4PtEntry[Index4].Bits.AddressHi);
 400     for (Index3 = 0; Index3 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index3++) {
 401       if (Lvl3PtEntry[Index3].Uint64 != 0) {
 402         DEBUG ((DEBUG_VERBOSE,"    Lvl3Pt Entry(0x%03x) - 0x%016lx\n", Index3, Lvl3PtEntry[Index3].Uint64));
 403       }
 404       if (Lvl3PtEntry[Index3].Uint64 == 0) {
 405         continue;
 406       }
 407
 408       Lvl2PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl3PtEntry[Index3].Bits.AddressLo, Lvl3PtEntry[Index3].Bits.AddressHi);
 409       for (Index2 = 0; Index2 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index2++) {
 410         if (Lvl2PtEntry[Index2].Uint64 != 0) {
 411           DEBUG ((DEBUG_VERBOSE,"      Lvl2Pt Entry(0x%03x) - 0x%016lx\n", Index2, Lvl2PtEntry[Index2].Uint64));
 412         }
 413         if (Lvl2PtEntry[Index2].Uint64 == 0) {
 414           continue;
 415         }
 416         if (Lvl2PtEntry[Index2].Bits.PageSize == 0) {
 417           Lvl1PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl2PtEntry[Index2].Bits.AddressLo, Lvl2PtEntry[Index2].Bits.AddressHi);
 418           for (Index1 = 0; Index1 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index1++) {
 419             if (Lvl1PtEntry[Index1].Uint64 != 0) {
 420               DEBUG ((DEBUG_VERBOSE,"        Lvl1Pt Entry(0x%03x) - 0x%016lx\n", Index1, Lvl1PtEntry[Index1].Uint64));
 421             }
 422           }
 423         }
 424       }
 425     }
 426   }
 427   DEBUG ((DEBUG_VERBOSE,"================\n"));
 428 }
 429
 430 /**
 431   Invalid page entry.
 432
 433   @param VtdIndex  The VTd engine index.
 434 **/
 435 VOID
 436 InvalidatePageEntry (
 437   IN UINTN                 VtdIndex
 438   )
 439 {
 440   if (mVtdUnitInformation[VtdIndex].HasDirtyContext || mVtdUnitInformation[VtdIndex].HasDirtyPages) {
 441     InvalidateVtdIOTLBGlobal (VtdIndex);
 442   }
 443   mVtdUnitInformation[VtdIndex].HasDirtyContext = FALSE;
 444   mVtdUnitInformation[VtdIndex].HasDirtyPages = FALSE;
 445 }
 446
 447 #define VTD_PG_R                   BIT0
 448 #define VTD_PG_W                   BIT1
 449 #define VTD_PG_X                   BIT2
 450 #define VTD_PG_EMT                 (BIT3 | BIT4 | BIT5)
 451 #define VTD_PG_TM                  (BIT62)
 452
 453 #define VTD_PG_PS                  BIT7
 454
 455 #define PAGE_PROGATE_BITS          (VTD_PG_TM | VTD_PG_EMT | VTD_PG_W | VTD_PG_R)
 456
 457 #define PAGING_4K_MASK  0xFFF
 458 #define PAGING_2M_MASK  0x1FFFFF
 459 #define PAGING_1G_MASK  0x3FFFFFFF
 460
 461 #define PAGING_VTD_INDEX_MASK     0x1FF
 462
 463 #define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull
 464 #define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull
 465 #define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull
 466
 467 typedef enum {
 468   PageNone,
 469   Page4K,
 470   Page2M,
 471   Page1G,
 472 } PAGE_ATTRIBUTE;
 473
 474 typedef struct {
 475   PAGE_ATTRIBUTE   Attribute;
 476   UINT64           Length;
 477   UINT64           AddressMask;
 478 } PAGE_ATTRIBUTE_TABLE;
 479
 480 PAGE_ATTRIBUTE_TABLE mPageAttributeTable[] = {
 481   {Page4K,  SIZE_4KB, PAGING_4K_ADDRESS_MASK_64},
 482   {Page2M,  SIZE_2MB, PAGING_2M_ADDRESS_MASK_64},
 483   {Page1G,  SIZE_1GB, PAGING_1G_ADDRESS_MASK_64},
 484 };
 485
 486 /**
 487   Return length according to page attributes.
 488
 489   @param[in]  PageAttributes   The page attribute of the page entry.
 490
 491   @return The length of page entry.
 492 **/
 493 UINTN
 494 PageAttributeToLength (
 495   IN PAGE_ATTRIBUTE  PageAttribute
 496   )
 497 {
 498   UINTN  Index;
 499   for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {
 500     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 501       return (UINTN)mPageAttributeTable[Index].Length;
 502     }
 503   }
 504   return 0;
 505 }
 506
 507 /**
 508   Return page table entry to match the address.
 509
 510   @param[in]   VtdIndex                 The index used to identify a VTd engine.
 511   @param[in]   SecondLevelPagingEntry   The second level paging entry in VTd table for the device.
 512   @param[in]   Address                  The address to be checked.
 513   @param[out]  PageAttributes           The page attribute of the page entry.
 514
 515   @return The page entry.
 516 **/
 517 VOID *
 518 GetSecondLevelPageTableEntry (
 519   IN  UINTN                         VtdIndex,
 520   IN  VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry,
 521   IN  PHYSICAL_ADDRESS              Address,
 522   OUT PAGE_ATTRIBUTE                *PageAttribute
 523   )
 524 {
 525   UINTN                 Index1;
 526   UINTN                 Index2;
 527   UINTN                 Index3;
 528   UINTN                 Index4;
 529   UINT64                *L1PageTable;
 530   UINT64                *L2PageTable;
 531   UINT64                *L3PageTable;
 532   UINT64                *L4PageTable;
 533
 534   Index4 = ((UINTN)RShiftU64 (Address, 39)) & PAGING_VTD_INDEX_MASK;
 535   Index3 = ((UINTN)Address >> 30) & PAGING_VTD_INDEX_MASK;
 536   Index2 = ((UINTN)Address >> 21) & PAGING_VTD_INDEX_MASK;
 537   Index1 = ((UINTN)Address >> 12) & PAGING_VTD_INDEX_MASK;
 538
 539   L4PageTable = (UINT64 *)SecondLevelPagingEntry;
 540   if (L4PageTable[Index4] == 0) {
 541     L4PageTable[Index4] = (UINT64)(UINTN)AllocateZeroPages (1);
 542     if (L4PageTable[Index4] == 0) {
 543       DEBUG ((DEBUG_ERROR,"!!!!!! ALLOCATE LVL4 PAGE FAIL (0x%x)!!!!!!\n", Index4));
 544       ASSERT(FALSE);
 545       *PageAttribute = PageNone;
 546       return NULL;
 547     }
 548     FlushPageTableMemory (VtdIndex, (UINTN)L4PageTable[Index4], SIZE_4KB);
 549     SetSecondLevelPagingEntryAttribute ((VTD_SECOND_LEVEL_PAGING_ENTRY *)&L4PageTable[Index4], EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 550     FlushPageTableMemory (VtdIndex, (UINTN)&L4PageTable[Index4], sizeof(L4PageTable[Index4]));
 551   }
 552
 553   L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & PAGING_4K_ADDRESS_MASK_64);
 554   if (L3PageTable[Index3] == 0) {
 555     L3PageTable[Index3] = (UINT64)(UINTN)AllocateZeroPages (1);
 556     if (L3PageTable[Index3] == 0) {
 557       DEBUG ((DEBUG_ERROR,"!!!!!! ALLOCATE LVL3 PAGE FAIL (0x%x, 0x%x)!!!!!!\n", Index4, Index3));
 558       ASSERT(FALSE);
 559       *PageAttribute = PageNone;
 560       return NULL;
 561     }
 562     FlushPageTableMemory (VtdIndex, (UINTN)L3PageTable[Index3], SIZE_4KB);
 563     SetSecondLevelPagingEntryAttribute ((VTD_SECOND_LEVEL_PAGING_ENTRY *)&L3PageTable[Index3], EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 564     FlushPageTableMemory (VtdIndex, (UINTN)&L3PageTable[Index3], sizeof(L3PageTable[Index3]));
 565   }
 566   if ((L3PageTable[Index3] & VTD_PG_PS) != 0) {
 567     // 1G
 568     *PageAttribute = Page1G;
 569     return &L3PageTable[Index3];
 570   }
 571
 572   L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & PAGING_4K_ADDRESS_MASK_64);
 573   if (L2PageTable[Index2] == 0) {
 574     L2PageTable[Index2] = Address & PAGING_2M_ADDRESS_MASK_64;
 575     SetSecondLevelPagingEntryAttribute ((VTD_SECOND_LEVEL_PAGING_ENTRY *)&L2PageTable[Index2], 0);
 576     L2PageTable[Index2] |= VTD_PG_PS;
 577     FlushPageTableMemory (VtdIndex, (UINTN)&L2PageTable[Index2], sizeof(L2PageTable[Index2]));
 578   }
 579   if ((L2PageTable[Index2] & VTD_PG_PS) != 0) {
 580     // 2M
 581     *PageAttribute = Page2M;
 582     return &L2PageTable[Index2];
 583   }
 584
 585   // 4k
 586   L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & PAGING_4K_ADDRESS_MASK_64);
 587   if ((L1PageTable[Index1] == 0) && (Address != 0)) {
 588     *PageAttribute = PageNone;
 589     return NULL;
 590   }
 591   *PageAttribute = Page4K;
 592   return &L1PageTable[Index1];
 593 }
 594
 595 /**
 596   Modify memory attributes of page entry.
 597
 598   @param[in]   VtdIndex         The index used to identify a VTd engine.
 599   @param[in]   PageEntry        The page entry.
 600   @param[in]   IoMmuAccess      The IOMMU access.
 601   @param[out]  IsModified       TRUE means page table modified. FALSE means page table not modified.
 602 **/
 603 VOID
 604 ConvertSecondLevelPageEntryAttribute (
 605   IN  UINTN                             VtdIndex,
 606   IN  VTD_SECOND_LEVEL_PAGING_ENTRY     *PageEntry,
 607   IN  UINT64                            IoMmuAccess,
 608   OUT BOOLEAN                           *IsModified
 609   )
 610 {
 611   UINT64  CurrentPageEntry;
 612   UINT64  NewPageEntry;
 613
 614   CurrentPageEntry = PageEntry->Uint64;
 615   SetSecondLevelPagingEntryAttribute (PageEntry, IoMmuAccess);
 616   FlushPageTableMemory (VtdIndex, (UINTN)PageEntry, sizeof(*PageEntry));
 617   NewPageEntry = PageEntry->Uint64;
 618   if (CurrentPageEntry != NewPageEntry) {
 619     *IsModified = TRUE;
 620     DEBUG ((DEBUG_VERBOSE, "ConvertSecondLevelPageEntryAttribute 0x%lx", CurrentPageEntry));
 621     DEBUG ((DEBUG_VERBOSE, "->0x%lx\n", NewPageEntry));
 622   } else {
 623     *IsModified = FALSE;
 624   }
 625 }
 626
 627 /**
 628   This function returns if there is need to split page entry.
 629
 630   @param[in]  BaseAddress      The base address to be checked.
 631   @param[in]  Length           The length to be checked.
 632   @param[in]  PageAttribute    The page attribute of the page entry.
 633
 634   @retval SplitAttributes on if there is need to split page entry.
 635 **/
 636 PAGE_ATTRIBUTE
 637 NeedSplitPage (
 638   IN  PHYSICAL_ADDRESS                  BaseAddress,
 639   IN  UINT64                            Length,
 640   IN  PAGE_ATTRIBUTE                    PageAttribute
 641   )
 642 {
 643   UINT64                PageEntryLength;
 644
 645   PageEntryLength = PageAttributeToLength (PageAttribute);
 646
 647   if (((BaseAddress & (PageEntryLength - 1)) == 0) && (Length >= PageEntryLength)) {
 648     return PageNone;
 649   }
 650
 651   if (((BaseAddress & PAGING_2M_MASK) != 0) || (Length < SIZE_2MB)) {
 652     return Page4K;
 653   }
 654
 655   return Page2M;
 656 }
 657
 658 /**
 659   This function splits one page entry to small page entries.
 660
 661   @param[in]  VtdIndex         The index used to identify a VTd engine.
 662   @param[in]  PageEntry        The page entry to be splitted.
 663   @param[in]  PageAttribute    The page attribute of the page entry.
 664   @param[in]  SplitAttribute   How to split the page entry.
 665
 666   @retval RETURN_SUCCESS            The page entry is splitted.
 667   @retval RETURN_UNSUPPORTED        The page entry does not support to be splitted.
 668   @retval RETURN_OUT_OF_RESOURCES   No resource to split page entry.
 669 **/
 670 RETURN_STATUS
 671 SplitSecondLevelPage (
 672   IN  UINTN                             VtdIndex,
 673   IN  VTD_SECOND_LEVEL_PAGING_ENTRY     *PageEntry,
 674   IN  PAGE_ATTRIBUTE                    PageAttribute,
 675   IN  PAGE_ATTRIBUTE                    SplitAttribute
 676   )
 677 {
 678   UINT64   BaseAddress;
 679   UINT64   *NewPageEntry;
 680   UINTN    Index;
 681
 682   ASSERT (PageAttribute == Page2M || PageAttribute == Page1G);
 683
 684   if (PageAttribute == Page2M) {
 685     //
 686     // Split 2M to 4K
 687     //
 688     ASSERT (SplitAttribute == Page4K);
 689     if (SplitAttribute == Page4K) {
 690       NewPageEntry = AllocateZeroPages (1);
 691       DEBUG ((DEBUG_VERBOSE, "Split - 0x%x\n", NewPageEntry));
 692       if (NewPageEntry == NULL) {
 693         return RETURN_OUT_OF_RESOURCES;
 694       }
 695       BaseAddress = PageEntry->Uint64 & PAGING_2M_ADDRESS_MASK_64;
 696       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 697         NewPageEntry[Index] = (BaseAddress + SIZE_4KB * Index) | (PageEntry->Uint64 & PAGE_PROGATE_BITS);
 698       }
 699       FlushPageTableMemory (VtdIndex, (UINTN)NewPageEntry, SIZE_4KB);
 700
 701       PageEntry->Uint64 = (UINT64)(UINTN)NewPageEntry;
 702       SetSecondLevelPagingEntryAttribute (PageEntry, EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 703       FlushPageTableMemory (VtdIndex, (UINTN)PageEntry, sizeof(*PageEntry));
 704       return RETURN_SUCCESS;
 705     } else {
 706       return RETURN_UNSUPPORTED;
 707     }
 708   } else if (PageAttribute == Page1G) {
 709     //
 710     // Split 1G to 2M
 711     // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
 712     //
 713     ASSERT (SplitAttribute == Page2M || SplitAttribute == Page4K);
 714     if ((SplitAttribute == Page2M || SplitAttribute == Page4K)) {
 715       NewPageEntry = AllocateZeroPages (1);
 716       DEBUG ((DEBUG_VERBOSE, "Split - 0x%x\n", NewPageEntry));
 717       if (NewPageEntry == NULL) {
 718         return RETURN_OUT_OF_RESOURCES;
 719       }
 720       BaseAddress = PageEntry->Uint64 & PAGING_1G_ADDRESS_MASK_64;
 721       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 722         NewPageEntry[Index] = (BaseAddress + SIZE_2MB * Index) | VTD_PG_PS | (PageEntry->Uint64 & PAGE_PROGATE_BITS);
 723       }
 724       FlushPageTableMemory (VtdIndex, (UINTN)NewPageEntry, SIZE_4KB);
 725
 726       PageEntry->Uint64 = (UINT64)(UINTN)NewPageEntry;
 727       SetSecondLevelPagingEntryAttribute (PageEntry, EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 728       FlushPageTableMemory (VtdIndex, (UINTN)PageEntry, sizeof(*PageEntry));
 729       return RETURN_SUCCESS;
 730     } else {
 731       return RETURN_UNSUPPORTED;
 732     }
 733   } else {
 734     return RETURN_UNSUPPORTED;
 735   }
 736 }
 737
 738 /**
 739   Set VTd attribute for a system memory on second level page entry
 740
 741   @param[in]  VtdIndex                The index used to identify a VTd engine.
 742   @param[in]  DomainIdentifier        The domain ID of the source.
 743   @param[in]  SecondLevelPagingEntry  The second level paging entry in VTd table for the device.
 744   @param[in]  BaseAddress             The base of device memory address to be used as the DMA memory.
 745   @param[in]  Length                  The length of device memory address to be used as the DMA memory.
 746   @param[in]  IoMmuAccess             The IOMMU access.
 747
 748   @retval EFI_SUCCESS            The IoMmuAccess is set for the memory range specified by BaseAddress and Length.
 749   @retval EFI_INVALID_PARAMETER  BaseAddress is not IoMmu Page size aligned.
 750   @retval EFI_INVALID_PARAMETER  Length is not IoMmu Page size aligned.
 751   @retval EFI_INVALID_PARAMETER  Length is 0.
 752   @retval EFI_INVALID_PARAMETER  IoMmuAccess specified an illegal combination of access.
 753   @retval EFI_UNSUPPORTED        The bit mask of IoMmuAccess is not supported by the IOMMU.
 754   @retval EFI_UNSUPPORTED        The IOMMU does not support the memory range specified by BaseAddress and Length.
 755   @retval EFI_OUT_OF_RESOURCES   There are not enough resources available to modify the IOMMU access.
 756   @retval EFI_DEVICE_ERROR       The IOMMU device reported an error while attempting the operation.
 757 **/
 758 EFI_STATUS
 759 SetSecondLevelPagingAttribute (
 760   IN UINTN                         VtdIndex,
 761   IN UINT16                        DomainIdentifier,
 762   IN VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry,
 763   IN UINT64                        BaseAddress,
 764   IN UINT64                        Length,
 765   IN UINT64                        IoMmuAccess
 766   )
 767 {
 768   VTD_SECOND_LEVEL_PAGING_ENTRY  *PageEntry;
 769   PAGE_ATTRIBUTE                 PageAttribute;
 770   UINTN                          PageEntryLength;
 771   PAGE_ATTRIBUTE                 SplitAttribute;
 772   EFI_STATUS                     Status;
 773   BOOLEAN                        IsEntryModified;
 774
 775   DEBUG ((DEBUG_VERBOSE,"SetSecondLevelPagingAttribute (%d) (0x%016lx - 0x%016lx : %x) \n", VtdIndex, BaseAddress, Length, IoMmuAccess));
 776   DEBUG ((DEBUG_VERBOSE,"  SecondLevelPagingEntry Base - 0x%x\n", SecondLevelPagingEntry));
 777
 778   if (BaseAddress != ALIGN_VALUE(BaseAddress, SIZE_4KB)) {
 779     DEBUG ((DEBUG_ERROR, "SetSecondLevelPagingAttribute - Invalid Alignment\n"));
 780     return EFI_UNSUPPORTED;
 781   }
 782   if (Length != ALIGN_VALUE(Length, SIZE_4KB)) {
 783     DEBUG ((DEBUG_ERROR, "SetSecondLevelPagingAttribute - Invalid Alignment\n"));
 784     return EFI_UNSUPPORTED;
 785   }
 786
 787   while (Length != 0) {
 788     PageEntry = GetSecondLevelPageTableEntry (VtdIndex, SecondLevelPagingEntry, BaseAddress, &PageAttribute);
 789     if (PageEntry == NULL) {
 790       DEBUG ((DEBUG_ERROR, "PageEntry - NULL\n"));
 791       return RETURN_UNSUPPORTED;
 792     }
 793     PageEntryLength = PageAttributeToLength (PageAttribute);
 794     SplitAttribute = NeedSplitPage (BaseAddress, Length, PageAttribute);
 795     if (SplitAttribute == PageNone) {
 796       ConvertSecondLevelPageEntryAttribute (VtdIndex, PageEntry, IoMmuAccess, &IsEntryModified);
 797       if (IsEntryModified) {
 798         mVtdUnitInformation[VtdIndex].HasDirtyPages = TRUE;
 799       }
 800       //
 801       // Convert success, move to next
 802       //
 803       BaseAddress += PageEntryLength;
 804       Length -= PageEntryLength;
 805     } else {
 806       Status = SplitSecondLevelPage (VtdIndex, PageEntry, PageAttribute, SplitAttribute);
 807       if (RETURN_ERROR (Status)) {
 808         DEBUG ((DEBUG_ERROR, "SplitSecondLevelPage - %r\n", Status));
 809         return RETURN_UNSUPPORTED;
 810       }
 811       mVtdUnitInformation[VtdIndex].HasDirtyPages = TRUE;
 812       //
 813       // Just split current page
 814       // Convert success in next around
 815       //
 816     }
 817   }
 818
 819   return EFI_SUCCESS;
 820 }
 821
 822 /**
 823   Set VTd attribute for a system memory.
 824
 825   @param[in]  VtdIndex                The index used to identify a VTd engine.
 826   @param[in]  DomainIdentifier        The domain ID of the source.
 827   @param[in]  SecondLevelPagingEntry  The second level paging entry in VTd table for the device.
 828   @param[in]  BaseAddress             The base of device memory address to be used as the DMA memory.
 829   @param[in]  Length                  The length of device memory address to be used as the DMA memory.
 830   @param[in]  IoMmuAccess             The IOMMU access.
 831
 832   @retval EFI_SUCCESS            The IoMmuAccess is set for the memory range specified by BaseAddress and Length.
 833   @retval EFI_INVALID_PARAMETER  BaseAddress is not IoMmu Page size aligned.
 834   @retval EFI_INVALID_PARAMETER  Length is not IoMmu Page size aligned.
 835   @retval EFI_INVALID_PARAMETER  Length is 0.
 836   @retval EFI_INVALID_PARAMETER  IoMmuAccess specified an illegal combination of access.
 837   @retval EFI_UNSUPPORTED        The bit mask of IoMmuAccess is not supported by the IOMMU.
 838   @retval EFI_UNSUPPORTED        The IOMMU does not support the memory range specified by BaseAddress and Length.
 839   @retval EFI_OUT_OF_RESOURCES   There are not enough resources available to modify the IOMMU access.
 840   @retval EFI_DEVICE_ERROR       The IOMMU device reported an error while attempting the operation.
 841 **/
 842 EFI_STATUS
 843 SetPageAttribute (
 844   IN UINTN                         VtdIndex,
 845   IN UINT16                        DomainIdentifier,
 846   IN VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry,
 847   IN UINT64                        BaseAddress,
 848   IN UINT64                        Length,
 849   IN UINT64                        IoMmuAccess
 850   )
 851 {
 852   EFI_STATUS Status;
 853   Status = EFI_NOT_FOUND;
 854   if (SecondLevelPagingEntry != NULL) {
 855     Status = SetSecondLevelPagingAttribute (VtdIndex, DomainIdentifier, SecondLevelPagingEntry, BaseAddress, Length, IoMmuAccess);
 856   }
 857   return Status;
 858 }
 859
 860 /**
 861   Set VTd attribute for a system memory.
 862
 863   @param[in]  Segment           The Segment used to identify a VTd engine.
 864   @param[in]  SourceId          The SourceId used to identify a VTd engine and table entry.
 865   @param[in]  BaseAddress       The base of device memory address to be used as the DMA memory.
 866   @param[in]  Length            The length of device memory address to be used as the DMA memory.
 867   @param[in]  IoMmuAccess       The IOMMU access.
 868
 869   @retval EFI_SUCCESS            The IoMmuAccess is set for the memory range specified by BaseAddress and Length.
 870   @retval EFI_INVALID_PARAMETER  BaseAddress is not IoMmu Page size aligned.
 871   @retval EFI_INVALID_PARAMETER  Length is not IoMmu Page size aligned.
 872   @retval EFI_INVALID_PARAMETER  Length is 0.
 873   @retval EFI_INVALID_PARAMETER  IoMmuAccess specified an illegal combination of access.
 874   @retval EFI_UNSUPPORTED        The bit mask of IoMmuAccess is not supported by the IOMMU.
 875   @retval EFI_UNSUPPORTED        The IOMMU does not support the memory range specified by BaseAddress and Length.
 876   @retval EFI_OUT_OF_RESOURCES   There are not enough resources available to modify the IOMMU access.
 877   @retval EFI_DEVICE_ERROR       The IOMMU device reported an error while attempting the operation.
 878 **/
 879 EFI_STATUS
 880 SetAccessAttribute (
 881   IN UINT16                Segment,
 882   IN VTD_SOURCE_ID         SourceId,
 883   IN UINT64                BaseAddress,
 884   IN UINT64                Length,
 885   IN UINT64                IoMmuAccess
 886   )
 887 {
 888   UINTN                         VtdIndex;
 889   EFI_STATUS                    Status;
 890   VTD_EXT_CONTEXT_ENTRY         *ExtContextEntry;
 891   VTD_CONTEXT_ENTRY             *ContextEntry;
 892   VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry;
 893   UINT64                        Pt;
 894   UINTN                         PciDataIndex;
 895   UINT16                        DomainIdentifier;
 896
 897   SecondLevelPagingEntry = NULL;
 898
 899   DEBUG ((DEBUG_VERBOSE,"SetAccessAttribute (S%04x B%02x D%02x F%02x) (0x%016lx - 0x%08x, %x)\n", Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function, BaseAddress, (UINTN)Length, IoMmuAccess));
 900
 901   VtdIndex = FindVtdIndexByPciDevice (Segment, SourceId, &ExtContextEntry, &ContextEntry);
 902   if (VtdIndex == (UINTN)-1) {
 903     DEBUG ((DEBUG_ERROR,"SetAccessAttribute - Pci device (S%04x B%02x D%02x F%02x) not found!\n", Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 904     return EFI_DEVICE_ERROR;
 905   }
 906
 907   PciDataIndex = GetPciDataIndex (VtdIndex, Segment, SourceId);
 908   mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceData[PciDataIndex].AccessCount++;
 909   //
 910   // DomainId should not be 0.
 911   //
 912   DomainIdentifier = (UINT16)(PciDataIndex + 1);
 913
 914   if (ExtContextEntry != NULL) {
 915     if (ExtContextEntry->Bits.Present == 0) {
 916       SecondLevelPagingEntry = CreateSecondLevelPagingEntry (VtdIndex, 0);
 917       DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry - 0x%x (S%04x B%02x D%02x F%02x) New\n", SecondLevelPagingEntry, Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 918       Pt = (UINT64)RShiftU64 ((UINT64)(UINTN)SecondLevelPagingEntry, 12);
 919
 920       ExtContextEntry->Bits.SecondLevelPageTranslationPointerLo = (UINT32) Pt;
 921       ExtContextEntry->Bits.SecondLevelPageTranslationPointerHi = (UINT32) RShiftU64(Pt, 20);
 922       ExtContextEntry->Bits.DomainIdentifier = DomainIdentifier;
 923       ExtContextEntry->Bits.Present = 1;
 924       FlushPageTableMemory (VtdIndex, (UINTN)ExtContextEntry, sizeof(*ExtContextEntry));
 925       DumpDmarExtContextEntryTable (mVtdUnitInformation[VtdIndex].ExtRootEntryTable);
 926       mVtdUnitInformation[VtdIndex].HasDirtyContext = TRUE;
 927     } else {
 928       SecondLevelPagingEntry = (VOID *)(UINTN)VTD_64BITS_ADDRESS(ExtContextEntry->Bits.SecondLevelPageTranslationPointerLo, ExtContextEntry->Bits.SecondLevelPageTranslationPointerHi);
 929       DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry - 0x%x (S%04x B%02x D%02x F%02x)\n", SecondLevelPagingEntry, Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 930     }
 931   } else if (ContextEntry != NULL) {
 932     if (ContextEntry->Bits.Present == 0) {
 933       SecondLevelPagingEntry = CreateSecondLevelPagingEntry (VtdIndex, 0);
 934       DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry - 0x%x (S%04x B%02x D%02x F%02x) New\n", SecondLevelPagingEntry, Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 935       Pt = (UINT64)RShiftU64 ((UINT64)(UINTN)SecondLevelPagingEntry, 12);
 936
 937       ContextEntry->Bits.SecondLevelPageTranslationPointerLo = (UINT32) Pt;
 938       ContextEntry->Bits.SecondLevelPageTranslationPointerHi = (UINT32) RShiftU64(Pt, 20);
 939       ContextEntry->Bits.DomainIdentifier = DomainIdentifier;
 940       ContextEntry->Bits.Present = 1;
 941       FlushPageTableMemory (VtdIndex, (UINTN)ContextEntry, sizeof(*ContextEntry));
 942       DumpDmarContextEntryTable (mVtdUnitInformation[VtdIndex].RootEntryTable);
 943       mVtdUnitInformation[VtdIndex].HasDirtyContext = TRUE;
 944     } else {
 945       SecondLevelPagingEntry = (VOID *)(UINTN)VTD_64BITS_ADDRESS(ContextEntry->Bits.SecondLevelPageTranslationPointerLo, ContextEntry->Bits.SecondLevelPageTranslationPointerHi);
 946       DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry - 0x%x (S%04x B%02x D%02x F%02x)\n", SecondLevelPagingEntry, Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 947     }
 948   }
 949
 950   //
 951   // Do not update FixedSecondLevelPagingEntry
 952   //
 953   if (SecondLevelPagingEntry != mVtdUnitInformation[VtdIndex].FixedSecondLevelPagingEntry) {
 954     Status = SetPageAttribute (
 955                VtdIndex,
 956                DomainIdentifier,
 957                SecondLevelPagingEntry,
 958                BaseAddress,
 959                Length,
 960                IoMmuAccess
 961                );
 962     if (EFI_ERROR (Status)) {
 963       DEBUG ((DEBUG_ERROR,"SetPageAttribute - %r\n", Status));
 964       return Status;
 965     }
 966   }
 967
 968   InvalidatePageEntry (VtdIndex);
 969
 970   return EFI_SUCCESS;
 971 }
 972
 973 /**
 974   Always enable the VTd page attribute for the device.
 975
 976   @param[in]  Segment           The Segment used to identify a VTd engine.
 977   @param[in]  SourceId          The SourceId used to identify a VTd engine and table entry.
 978
 979   @retval EFI_SUCCESS           The VTd entry is updated to always enable all DMA access for the specific device.
 980 **/
 981 EFI_STATUS
 982 AlwaysEnablePageAttribute (
 983   IN UINT16                  Segment,
 984   IN VTD_SOURCE_ID           SourceId
 985   )
 986 {
 987   UINTN                         VtdIndex;
 988   VTD_EXT_CONTEXT_ENTRY         *ExtContextEntry;
 989   VTD_CONTEXT_ENTRY             *ContextEntry;
 990   VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry;
 991   UINT64                        Pt;
 992
 993   DEBUG ((DEBUG_INFO,"AlwaysEnablePageAttribute (S%04x B%02x D%02x F%02x)\n", Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 994
 995   VtdIndex = FindVtdIndexByPciDevice (Segment, SourceId, &ExtContextEntry, &ContextEntry);
 996   if (VtdIndex == (UINTN)-1) {
 997     DEBUG ((DEBUG_ERROR,"AlwaysEnablePageAttribute - Pci device (S%04x B%02x D%02x F%02x) not found!\n", Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 998     return EFI_DEVICE_ERROR;
 999   }
1000
1001   if (mVtdUnitInformation[VtdIndex].FixedSecondLevelPagingEntry == 0) {
1002     DEBUG((DEBUG_INFO, "CreateSecondLevelPagingEntry - %d\n", VtdIndex));
1003     mVtdUnitInformation[VtdIndex].FixedSecondLevelPagingEntry = CreateSecondLevelPagingEntry (VtdIndex, EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
1004   }
1005
1006   SecondLevelPagingEntry = mVtdUnitInformation[VtdIndex].FixedSecondLevelPagingEntry;
1007   Pt = (UINT64)RShiftU64 ((UINT64)(UINTN)SecondLevelPagingEntry, 12);
1008   if (ExtContextEntry != NULL) {
1009     ExtContextEntry->Bits.SecondLevelPageTranslationPointerLo = (UINT32) Pt;
1010     ExtContextEntry->Bits.SecondLevelPageTranslationPointerHi = (UINT32) RShiftU64(Pt, 20);
1011     ExtContextEntry->Bits.DomainIdentifier = ((1 << (UINT8)((UINTN)mVtdUnitInformation[VtdIndex].CapReg.Bits.ND * 2 + 4)) - 1);
1012     ExtContextEntry->Bits.Present = 1;
1013     FlushPageTableMemory (VtdIndex, (UINTN)ExtContextEntry, sizeof(*ExtContextEntry));
1014   } else if (ContextEntry != NULL) {
1015     ContextEntry->Bits.SecondLevelPageTranslationPointerLo = (UINT32) Pt;
1016     ContextEntry->Bits.SecondLevelPageTranslationPointerHi = (UINT32) RShiftU64(Pt, 20);
1017     ContextEntry->Bits.DomainIdentifier = ((1 << (UINT8)((UINTN)mVtdUnitInformation[VtdIndex].CapReg.Bits.ND * 2 + 4)) - 1);
1018     ContextEntry->Bits.Present = 1;
1019     FlushPageTableMemory (VtdIndex, (UINTN)ContextEntry, sizeof(*ContextEntry));
1020   }
1021
1022   return EFI_SUCCESS;
1023 }