IntelSiliconPkg/IntelVTdDxe/TranslationTable.c

   1 /** @file
   2
   3   Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
   4   This program and the accompanying materials
   5   are licensed and made available under the terms and conditions of the BSD License
   6   which accompanies this distribution.  The full text of the license may be found at
   7   http://opensource.org/licenses/bsd-license.php.
   8
   9   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  10   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  11
  12 **/
  13
  14 #include "DmaProtection.h"
  15
  16 /**
  17   Create extended context entry.
  18
  19   @param[in]  VtdIndex  The index of the VTd engine.
  20
  21   @retval EFI_SUCCESS          The extended context entry is created.
  22   @retval EFI_OUT_OF_RESOURCE  No enough resource to create extended context entry.
  23 **/
  24 EFI_STATUS
  25 CreateExtContextEntry (
  26   IN UINTN  VtdIndex
  27   );
  28
  29 /**
  30   Allocate zero pages.
  31
  32   @param[in]  Pages the number of pages.
  33
  34   @return the page address.
  35   @retval NULL No resource to allocate pages.
  36 **/
  37 VOID *
  38 EFIAPI
  39 AllocateZeroPages (
  40   IN UINTN  Pages
  41   )
  42 {
  43   VOID *Addr;
  44
  45   Addr = AllocatePages (Pages);
  46   if (Addr == NULL) {
  47     return NULL;
  48   }
  49   ZeroMem (Addr, EFI_PAGES_TO_SIZE(Pages));
  50   return Addr;
  51 }
  52
  53 /**
  54   Set second level paging entry attribute based upon IoMmuAccess.
  55
  56   @param[in]  PtEntry      The paging entry.
  57   @param[in]  IoMmuAccess  The IOMMU access.
  58 **/
  59 VOID
  60 SetSecondLevelPagingEntryAttribute (
  61   IN VTD_SECOND_LEVEL_PAGING_ENTRY  *PtEntry,
  62   IN UINT64                         IoMmuAccess
  63   )
  64 {
  65   PtEntry->Bits.Read  = ((IoMmuAccess & EDKII_IOMMU_ACCESS_READ) != 0);
  66   PtEntry->Bits.Write = ((IoMmuAccess & EDKII_IOMMU_ACCESS_WRITE) != 0);
  67 }
  68
  69 /**
  70   Create context entry.
  71
  72   @param[in]  VtdIndex  The index of the VTd engine.
  73
  74   @retval EFI_SUCCESS          The context entry is created.
  75   @retval EFI_OUT_OF_RESOURCE  No enough resource to create context entry.
  76 **/
  77 EFI_STATUS
  78 CreateContextEntry (
  79   IN UINTN  VtdIndex
  80   )
  81 {
  82   UINTN                  Index;
  83   VOID                   *Buffer;
  84   UINTN                  RootPages;
  85   UINTN                  ContextPages;
  86   VTD_ROOT_ENTRY         *RootEntry;
  87   VTD_CONTEXT_ENTRY      *ContextEntryTable;
  88   VTD_CONTEXT_ENTRY      *ContextEntry;
  89   VTD_SOURCE_ID          *PciSourceId;
  90   VTD_SOURCE_ID          SourceId;
  91   UINTN                  MaxBusNumber;
  92   UINTN                  EntryTablePages;
  93
  94   MaxBusNumber = 0;
  95   for (Index = 0; Index < mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceDataNumber; Index++) {
  96     PciSourceId = &mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceData[Index].PciSourceId;
  97     if (PciSourceId->Bits.Bus > MaxBusNumber) {
  98       MaxBusNumber = PciSourceId->Bits.Bus;
  99     }
 100   }
 101   DEBUG ((DEBUG_INFO,"  MaxBusNumber - 0x%x\n", MaxBusNumber));
 102
 103   RootPages = EFI_SIZE_TO_PAGES (sizeof (VTD_ROOT_ENTRY) * VTD_ROOT_ENTRY_NUMBER);
 104   ContextPages = EFI_SIZE_TO_PAGES (sizeof (VTD_CONTEXT_ENTRY) * VTD_CONTEXT_ENTRY_NUMBER);
 105   EntryTablePages = RootPages + ContextPages * (MaxBusNumber + 1);
 106   Buffer = AllocateZeroPages (EntryTablePages);
 107   if (Buffer == NULL) {
 108     DEBUG ((DEBUG_INFO,"Could not Alloc Root Entry Table.. \n"));
 109     return EFI_OUT_OF_RESOURCES;
 110   }
 111   mVtdUnitInformation[VtdIndex].RootEntryTable = (VTD_ROOT_ENTRY *)Buffer;
 112   Buffer = (UINT8 *)Buffer + EFI_PAGES_TO_SIZE (RootPages);
 113
 114   for (Index = 0; Index < mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceDataNumber; Index++) {
 115     PciSourceId = &mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceData[Index].PciSourceId;
 116
 117     SourceId.Bits.Bus = PciSourceId->Bits.Bus;
 118     SourceId.Bits.Device = PciSourceId->Bits.Device;
 119     SourceId.Bits.Function = PciSourceId->Bits.Function;
 120
 121     RootEntry = &mVtdUnitInformation[VtdIndex].RootEntryTable[SourceId.Index.RootIndex];
 122     if (RootEntry->Bits.Present == 0) {
 123       RootEntry->Bits.ContextTablePointerLo  = (UINT32) RShiftU64 ((UINT64)(UINTN)Buffer, 12);
 124       RootEntry->Bits.ContextTablePointerHi  = (UINT32) RShiftU64 ((UINT64)(UINTN)Buffer, 32);
 125       RootEntry->Bits.Present = 1;
 126       Buffer = (UINT8 *)Buffer + EFI_PAGES_TO_SIZE (ContextPages);
 127       FlushPageTableMemory (VtdIndex, (UINTN)RootEntry, sizeof(*RootEntry));
 128     }
 129
 130     ContextEntryTable = (VTD_CONTEXT_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(RootEntry->Bits.ContextTablePointerLo, RootEntry->Bits.ContextTablePointerHi) ;
 131     ContextEntry = &ContextEntryTable[SourceId.Index.ContextIndex];
 132     ContextEntry->Bits.TranslationType = 0;
 133     ContextEntry->Bits.FaultProcessingDisable = 0;
 134     ContextEntry->Bits.Present = 0;
 135
 136     DEBUG ((DEBUG_INFO,"Source: S%04x B%02x D%02x F%02x\n", mVtdUnitInformation[VtdIndex].Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 137
 138     switch (mVtdUnitInformation[VtdIndex].CapReg.Bits.SAGAW) {
 139     case BIT1:
 140       ContextEntry->Bits.AddressWidth = 0x1;
 141       break;
 142     case BIT2:
 143       ContextEntry->Bits.AddressWidth = 0x2;
 144       break;
 145     }
 146     FlushPageTableMemory (VtdIndex, (UINTN)ContextEntry, sizeof(*ContextEntry));
 147   }
 148
 149   return EFI_SUCCESS;
 150 }
 151
 152 /**
 153   Create second level paging entry table.
 154
 155   @param[in]  VtdIndex                    The index of the VTd engine.
 156   @param[in]  SecondLevelPagingEntry      The second level paging entry.
 157   @param[in]  MemoryBase                  The base of the memory.
 158   @param[in]  MemoryLimit                 The limit of the memory.
 159   @param[in]  IoMmuAccess                 The IOMMU access.
 160
 161   @return The second level paging entry.
 162 **/
 163 VTD_SECOND_LEVEL_PAGING_ENTRY *
 164 CreateSecondLevelPagingEntryTable (
 165   IN UINTN                         VtdIndex,
 166   IN VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry,
 167   IN UINT64                        MemoryBase,
 168   IN UINT64                        MemoryLimit,
 169   IN UINT64                        IoMmuAccess
 170   )
 171 {
 172   UINTN                          Index4;
 173   UINTN                          Index3;
 174   UINTN                          Index2;
 175   UINTN                          Lvl4Start;
 176   UINTN                          Lvl4End;
 177   UINTN                          Lvl3Start;
 178   UINTN                          Lvl3End;
 179   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl4PtEntry;
 180   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl3PtEntry;
 181   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl2PtEntry;
 182   UINT64                         BaseAddress;
 183   UINT64                         EndAddress;
 184
 185   if (MemoryLimit == 0) {
 186     return EFI_SUCCESS;
 187   }
 188
 189   BaseAddress = ALIGN_VALUE_LOW(MemoryBase, SIZE_2MB);
 190   EndAddress = ALIGN_VALUE_UP(MemoryLimit, SIZE_2MB);
 191   DEBUG ((DEBUG_INFO,"CreateSecondLevelPagingEntryTable: BaseAddress - 0x%016lx, EndAddress - 0x%016lx\n", BaseAddress, EndAddress));
 192
 193   if (SecondLevelPagingEntry == NULL) {
 194     SecondLevelPagingEntry = AllocateZeroPages (1);
 195     if (SecondLevelPagingEntry == NULL) {
 196       DEBUG ((DEBUG_ERROR,"Could not Alloc LVL4 PT. \n"));
 197       return NULL;
 198     }
 199   }
 200
 201   //
 202   // If no access is needed, just create not present entry.
 203   //
 204   if (IoMmuAccess == 0) {
 205     return SecondLevelPagingEntry;
 206   }
 207
 208   Lvl4Start = RShiftU64 (BaseAddress, 39) & 0x1FF;
 209   Lvl4End = RShiftU64 (EndAddress - 1, 39) & 0x1FF;
 210
 211   DEBUG ((DEBUG_INFO,"  Lvl4Start - 0x%x, Lvl4End - 0x%x\n", Lvl4Start, Lvl4End));
 212
 213   Lvl4PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)SecondLevelPagingEntry;
 214   for (Index4 = Lvl4Start; Index4 <= Lvl4End; Index4++) {
 215     if (Lvl4PtEntry[Index4].Uint64 == 0) {
 216       Lvl4PtEntry[Index4].Uint64 = (UINT64)(UINTN)AllocateZeroPages (1);
 217       if (Lvl4PtEntry[Index4].Uint64 == 0) {
 218         DEBUG ((DEBUG_ERROR,"!!!!!! ALLOCATE LVL4 PAGE FAIL (0x%x)!!!!!!\n", Index4));
 219         ASSERT(FALSE);
 220         return NULL;
 221       }
 222       SetSecondLevelPagingEntryAttribute (&Lvl4PtEntry[Index4], EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 223     }
 224
 225     Lvl3Start = RShiftU64 (BaseAddress, 30) & 0x1FF;
 226     if (ALIGN_VALUE_LOW(BaseAddress + SIZE_1GB, SIZE_1GB) <= EndAddress) {
 227       Lvl3End = SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY);
 228     } else {
 229       Lvl3End = RShiftU64 (EndAddress - 1, 30) & 0x1FF;
 230     }
 231     DEBUG ((DEBUG_INFO,"  Lvl4(0x%x): Lvl3Start - 0x%x, Lvl3End - 0x%x\n", Index4, Lvl3Start, Lvl3End));
 232
 233     Lvl3PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl4PtEntry[Index4].Bits.AddressLo, Lvl4PtEntry[Index4].Bits.AddressHi);
 234     for (Index3 = Lvl3Start; Index3 <= Lvl3End; Index3++) {
 235       if (Lvl3PtEntry[Index3].Uint64 == 0) {
 236         Lvl3PtEntry[Index3].Uint64 = (UINT64)(UINTN)AllocateZeroPages (1);
 237         if (Lvl3PtEntry[Index3].Uint64 == 0) {
 238           DEBUG ((DEBUG_ERROR,"!!!!!! ALLOCATE LVL3 PAGE FAIL (0x%x, 0x%x)!!!!!!\n", Index4, Index3));
 239           ASSERT(FALSE);
 240           return NULL;
 241         }
 242         SetSecondLevelPagingEntryAttribute (&Lvl3PtEntry[Index3], EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 243       }
 244
 245       Lvl2PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl3PtEntry[Index3].Bits.AddressLo, Lvl3PtEntry[Index3].Bits.AddressHi);
 246       for (Index2 = 0; Index2 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index2++) {
 247         Lvl2PtEntry[Index2].Uint64 = BaseAddress;
 248         SetSecondLevelPagingEntryAttribute (&Lvl2PtEntry[Index2], IoMmuAccess);
 249         Lvl2PtEntry[Index2].Bits.PageSize = 1;
 250         BaseAddress += SIZE_2MB;
 251         if (BaseAddress >= MemoryLimit) {
 252           goto Done;
 253         }
 254       }
 255       FlushPageTableMemory (VtdIndex, (UINTN)Lvl2PtEntry, SIZE_4KB);
 256     }
 257     FlushPageTableMemory (VtdIndex, (UINTN)&Lvl3PtEntry[Lvl3Start], (UINTN)&Lvl3PtEntry[Lvl3End + 1] - (UINTN)&Lvl3PtEntry[Lvl3Start]);
 258   }
 259   FlushPageTableMemory (VtdIndex, (UINTN)&Lvl4PtEntry[Lvl4Start], (UINTN)&Lvl4PtEntry[Lvl4End + 1] - (UINTN)&Lvl4PtEntry[Lvl4Start]);
 260
 261 Done:
 262   return SecondLevelPagingEntry;
 263 }
 264
 265 /**
 266   Create second level paging entry.
 267
 268   @param[in]  VtdIndex                    The index of the VTd engine.
 269   @param[in]  IoMmuAccess                 The IOMMU access.
 270
 271   @return The second level paging entry.
 272 **/
 273 VTD_SECOND_LEVEL_PAGING_ENTRY *
 274 CreateSecondLevelPagingEntry (
 275   IN UINTN   VtdIndex,
 276   IN UINT64  IoMmuAccess
 277   )
 278 {
 279   VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry;
 280
 281   SecondLevelPagingEntry = NULL;
 282   SecondLevelPagingEntry = CreateSecondLevelPagingEntryTable (VtdIndex, SecondLevelPagingEntry, 0, mBelow4GMemoryLimit, IoMmuAccess);
 283   if (SecondLevelPagingEntry == NULL) {
 284     return NULL;
 285   }
 286   SecondLevelPagingEntry = CreateSecondLevelPagingEntryTable (VtdIndex, SecondLevelPagingEntry, SIZE_4GB, mAbove4GMemoryLimit, IoMmuAccess);
 287   if (SecondLevelPagingEntry == NULL) {
 288     return NULL;
 289   }
 290
 291   return SecondLevelPagingEntry;
 292 }
 293
 294 /**
 295   Setup VTd translation table.
 296
 297   @retval EFI_SUCCESS           Setup translation table successfully.
 298   @retval EFI_OUT_OF_RESOURCE   Setup translation table fail.
 299 **/
 300 EFI_STATUS
 301 SetupTranslationTable (
 302   VOID
 303   )
 304 {
 305   EFI_STATUS      Status;
 306   UINTN           Index;
 307
 308   for (Index = 0; Index < mVtdUnitNumber; Index++) {
 309     DEBUG((DEBUG_INFO, "CreateContextEntry - %d\n", Index));
 310     if (mVtdUnitInformation[Index].ECapReg.Bits.ECS) {
 311       Status = CreateExtContextEntry (Index);
 312     } else {
 313       Status = CreateContextEntry (Index);
 314     }
 315     if (EFI_ERROR (Status)) {
 316       return Status;
 317     }
 318   }
 319
 320   return EFI_SUCCESS;
 321 }
 322
 323 /**
 324   Dump DMAR context entry table.
 325
 326   @param[in]  RootEntry DMAR root entry.
 327 **/
 328 VOID
 329 DumpDmarContextEntryTable (
 330   IN VTD_ROOT_ENTRY *RootEntry
 331   )
 332 {
 333   UINTN                 Index;
 334   UINTN                 Index2;
 335   VTD_CONTEXT_ENTRY     *ContextEntry;
 336
 337   DEBUG ((DEBUG_INFO,"=========================\n"));
 338   DEBUG ((DEBUG_INFO,"DMAR Context Entry Table:\n"));
 339
 340   DEBUG ((DEBUG_INFO,"RootEntry Address - 0x%x\n", RootEntry));
 341
 342   for (Index = 0; Index < VTD_ROOT_ENTRY_NUMBER; Index++) {
 343     if ((RootEntry[Index].Uint128.Uint64Lo != 0) || (RootEntry[Index].Uint128.Uint64Hi != 0)) {
 344       DEBUG ((DEBUG_INFO,"  RootEntry(0x%02x) B%02x - 0x%016lx %016lx\n",
 345         Index, Index, RootEntry[Index].Uint128.Uint64Hi, RootEntry[Index].Uint128.Uint64Lo));
 346     }
 347     if (RootEntry[Index].Bits.Present == 0) {
 348       continue;
 349     }
 350     ContextEntry = (VTD_CONTEXT_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(RootEntry[Index].Bits.ContextTablePointerLo, RootEntry[Index].Bits.ContextTablePointerHi);
 351     for (Index2 = 0; Index2 < VTD_CONTEXT_ENTRY_NUMBER; Index2++) {
 352       if ((ContextEntry[Index2].Uint128.Uint64Lo != 0) || (ContextEntry[Index2].Uint128.Uint64Hi != 0)) {
 353         DEBUG ((DEBUG_INFO,"    ContextEntry(0x%02x) D%02xF%02x - 0x%016lx %016lx\n",
 354           Index2, Index2 >> 3, Index2 & 0x7, ContextEntry[Index2].Uint128.Uint64Hi, ContextEntry[Index2].Uint128.Uint64Lo));
 355       }
 356       if (ContextEntry[Index2].Bits.Present == 0) {
 357         continue;
 358       }
 359       DumpSecondLevelPagingEntry ((VOID *)(UINTN)VTD_64BITS_ADDRESS(ContextEntry[Index2].Bits.SecondLevelPageTranslationPointerLo, ContextEntry[Index2].Bits.SecondLevelPageTranslationPointerHi));
 360     }
 361   }
 362   DEBUG ((DEBUG_INFO,"=========================\n"));
 363 }
 364
 365 /**
 366   Dump DMAR second level paging entry.
 367
 368   @param[in]  SecondLevelPagingEntry The second level paging entry.
 369 **/
 370 VOID
 371 DumpSecondLevelPagingEntry (
 372   IN VOID *SecondLevelPagingEntry
 373   )
 374 {
 375   UINTN                          Index4;
 376   UINTN                          Index3;
 377   UINTN                          Index2;
 378   UINTN                          Index1;
 379   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl4PtEntry;
 380   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl3PtEntry;
 381   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl2PtEntry;
 382   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl1PtEntry;
 383
 384   DEBUG ((DEBUG_VERBOSE,"================\n"));
 385   DEBUG ((DEBUG_VERBOSE,"DMAR Second Level Page Table:\n"));
 386
 387   DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry Base - 0x%x\n", SecondLevelPagingEntry));
 388   Lvl4PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)SecondLevelPagingEntry;
 389   for (Index4 = 0; Index4 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index4++) {
 390     if (Lvl4PtEntry[Index4].Uint64 != 0) {
 391       DEBUG ((DEBUG_VERBOSE,"  Lvl4Pt Entry(0x%03x) - 0x%016lx\n", Index4, Lvl4PtEntry[Index4].Uint64));
 392     }
 393     if (Lvl4PtEntry[Index4].Uint64 == 0) {
 394       continue;
 395     }
 396     Lvl3PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl4PtEntry[Index4].Bits.AddressLo, Lvl4PtEntry[Index4].Bits.AddressHi);
 397     for (Index3 = 0; Index3 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index3++) {
 398       if (Lvl3PtEntry[Index3].Uint64 != 0) {
 399         DEBUG ((DEBUG_VERBOSE,"    Lvl3Pt Entry(0x%03x) - 0x%016lx\n", Index3, Lvl3PtEntry[Index3].Uint64));
 400       }
 401       if (Lvl3PtEntry[Index3].Uint64 == 0) {
 402         continue;
 403       }
 404
 405       Lvl2PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl3PtEntry[Index3].Bits.AddressLo, Lvl3PtEntry[Index3].Bits.AddressHi);
 406       for (Index2 = 0; Index2 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index2++) {
 407         if (Lvl2PtEntry[Index2].Uint64 != 0) {
 408           DEBUG ((DEBUG_VERBOSE,"      Lvl2Pt Entry(0x%03x) - 0x%016lx\n", Index2, Lvl2PtEntry[Index2].Uint64));
 409         }
 410         if (Lvl2PtEntry[Index2].Uint64 == 0) {
 411           continue;
 412         }
 413         if (Lvl2PtEntry[Index2].Bits.PageSize == 0) {
 414           Lvl1PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl2PtEntry[Index2].Bits.AddressLo, Lvl2PtEntry[Index2].Bits.AddressHi);
 415           for (Index1 = 0; Index1 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index1++) {
 416             if (Lvl1PtEntry[Index1].Uint64 != 0) {
 417               DEBUG ((DEBUG_VERBOSE,"        Lvl1Pt Entry(0x%03x) - 0x%016lx\n", Index1, Lvl1PtEntry[Index1].Uint64));
 418             }
 419           }
 420         }
 421       }
 422     }
 423   }
 424   DEBUG ((DEBUG_VERBOSE,"================\n"));
 425 }
 426
 427 /**
 428   Invalid page entry.
 429
 430   @param VtdIndex  The VTd engine index.
 431 **/
 432 VOID
 433 InvalidatePageEntry (
 434   IN UINTN                 VtdIndex
 435   )
 436 {
 437   if (mVtdUnitInformation[VtdIndex].HasDirtyContext || mVtdUnitInformation[VtdIndex].HasDirtyPages) {
 438     InvalidateVtdIOTLBGlobal (VtdIndex);
 439   }
 440   mVtdUnitInformation[VtdIndex].HasDirtyContext = FALSE;
 441   mVtdUnitInformation[VtdIndex].HasDirtyPages = FALSE;
 442 }
 443
 444 #define VTD_PG_R                   BIT0
 445 #define VTD_PG_W                   BIT1
 446 #define VTD_PG_X                   BIT2
 447 #define VTD_PG_EMT                 (BIT3 | BIT4 | BIT5)
 448 #define VTD_PG_TM                  (BIT62)
 449
 450 #define VTD_PG_PS                  BIT7
 451
 452 #define PAGE_PROGATE_BITS          (VTD_PG_TM | VTD_PG_EMT | VTD_PG_W | VTD_PG_R)
 453
 454 #define PAGING_4K_MASK  0xFFF
 455 #define PAGING_2M_MASK  0x1FFFFF
 456 #define PAGING_1G_MASK  0x3FFFFFFF
 457
 458 #define PAGING_VTD_INDEX_MASK     0x1FF
 459
 460 #define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull
 461 #define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull
 462 #define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull
 463
 464 typedef enum {
 465   PageNone,
 466   Page4K,
 467   Page2M,
 468   Page1G,
 469 } PAGE_ATTRIBUTE;
 470
 471 typedef struct {
 472   PAGE_ATTRIBUTE   Attribute;
 473   UINT64           Length;
 474   UINT64           AddressMask;
 475 } PAGE_ATTRIBUTE_TABLE;
 476
 477 PAGE_ATTRIBUTE_TABLE mPageAttributeTable[] = {
 478   {Page4K,  SIZE_4KB, PAGING_4K_ADDRESS_MASK_64},
 479   {Page2M,  SIZE_2MB, PAGING_2M_ADDRESS_MASK_64},
 480   {Page1G,  SIZE_1GB, PAGING_1G_ADDRESS_MASK_64},
 481 };
 482
 483 /**
 484   Return length according to page attributes.
 485
 486   @param[in]  PageAttributes   The page attribute of the page entry.
 487
 488   @return The length of page entry.
 489 **/
 490 UINTN
 491 PageAttributeToLength (
 492   IN PAGE_ATTRIBUTE  PageAttribute
 493   )
 494 {
 495   UINTN  Index;
 496   for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {
 497     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 498       return (UINTN)mPageAttributeTable[Index].Length;
 499     }
 500   }
 501   return 0;
 502 }
 503
 504 /**
 505   Return page table entry to match the address.
 506
 507   @param[in]   VtdIndex                 The index used to identify a VTd engine.
 508   @param[in]   SecondLevelPagingEntry   The second level paging entry in VTd table for the device.
 509   @param[in]   Address                  The address to be checked.
 510   @param[out]  PageAttributes           The page attribute of the page entry.
 511
 512   @return The page entry.
 513 **/
 514 VOID *
 515 GetSecondLevelPageTableEntry (
 516   IN  UINTN                         VtdIndex,
 517   IN  VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry,
 518   IN  PHYSICAL_ADDRESS              Address,
 519   OUT PAGE_ATTRIBUTE                *PageAttribute
 520   )
 521 {
 522   UINTN                 Index1;
 523   UINTN                 Index2;
 524   UINTN                 Index3;
 525   UINTN                 Index4;
 526   UINT64                *L1PageTable;
 527   UINT64                *L2PageTable;
 528   UINT64                *L3PageTable;
 529   UINT64                *L4PageTable;
 530
 531   Index4 = ((UINTN)RShiftU64 (Address, 39)) & PAGING_VTD_INDEX_MASK;
 532   Index3 = ((UINTN)Address >> 30) & PAGING_VTD_INDEX_MASK;
 533   Index2 = ((UINTN)Address >> 21) & PAGING_VTD_INDEX_MASK;
 534   Index1 = ((UINTN)Address >> 12) & PAGING_VTD_INDEX_MASK;
 535
 536   L4PageTable = (UINT64 *)SecondLevelPagingEntry;
 537   if (L4PageTable[Index4] == 0) {
 538     L4PageTable[Index4] = (UINT64)(UINTN)AllocateZeroPages (1);
 539     if (L4PageTable[Index4] == 0) {
 540       DEBUG ((DEBUG_ERROR,"!!!!!! ALLOCATE LVL4 PAGE FAIL (0x%x)!!!!!!\n", Index4));
 541       ASSERT(FALSE);
 542       *PageAttribute = PageNone;
 543       return NULL;
 544     }
 545     SetSecondLevelPagingEntryAttribute ((VTD_SECOND_LEVEL_PAGING_ENTRY *)&L4PageTable[Index4], EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 546     FlushPageTableMemory (VtdIndex, (UINTN)&L4PageTable[Index4], sizeof(L4PageTable[Index4]));
 547   }
 548
 549   L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & PAGING_4K_ADDRESS_MASK_64);
 550   if (L3PageTable[Index3] == 0) {
 551     L3PageTable[Index3] = (UINT64)(UINTN)AllocateZeroPages (1);
 552     if (L3PageTable[Index3] == 0) {
 553       DEBUG ((DEBUG_ERROR,"!!!!!! ALLOCATE LVL3 PAGE FAIL (0x%x, 0x%x)!!!!!!\n", Index4, Index3));
 554       ASSERT(FALSE);
 555       *PageAttribute = PageNone;
 556       return NULL;
 557     }
 558     SetSecondLevelPagingEntryAttribute ((VTD_SECOND_LEVEL_PAGING_ENTRY *)&L3PageTable[Index3], EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 559     FlushPageTableMemory (VtdIndex, (UINTN)&L3PageTable[Index3], sizeof(L3PageTable[Index3]));
 560   }
 561   if ((L3PageTable[Index3] & VTD_PG_PS) != 0) {
 562     // 1G
 563     *PageAttribute = Page1G;
 564     return &L3PageTable[Index3];
 565   }
 566
 567   L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & PAGING_4K_ADDRESS_MASK_64);
 568   if (L2PageTable[Index2] == 0) {
 569     L2PageTable[Index2] = Address & PAGING_2M_ADDRESS_MASK_64;
 570     SetSecondLevelPagingEntryAttribute ((VTD_SECOND_LEVEL_PAGING_ENTRY *)&L2PageTable[Index2], 0);
 571     L2PageTable[Index2] |= VTD_PG_PS;
 572     FlushPageTableMemory (VtdIndex, (UINTN)&L2PageTable[Index2], sizeof(L2PageTable[Index2]));
 573   }
 574   if ((L2PageTable[Index2] & VTD_PG_PS) != 0) {
 575     // 2M
 576     *PageAttribute = Page2M;
 577     return &L2PageTable[Index2];
 578   }
 579
 580   // 4k
 581   L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & PAGING_4K_ADDRESS_MASK_64);
 582   if ((L1PageTable[Index1] == 0) && (Address != 0)) {
 583     *PageAttribute = PageNone;
 584     return NULL;
 585   }
 586   *PageAttribute = Page4K;
 587   return &L1PageTable[Index1];
 588 }
 589
 590 /**
 591   Modify memory attributes of page entry.
 592
 593   @param[in]   VtdIndex         The index used to identify a VTd engine.
 594   @param[in]   PageEntry        The page entry.
 595   @param[in]   IoMmuAccess      The IOMMU access.
 596   @param[out]  IsModified       TRUE means page table modified. FALSE means page table not modified.
 597 **/
 598 VOID
 599 ConvertSecondLevelPageEntryAttribute (
 600   IN  UINTN                             VtdIndex,
 601   IN  VTD_SECOND_LEVEL_PAGING_ENTRY     *PageEntry,
 602   IN  UINT64                            IoMmuAccess,
 603   OUT BOOLEAN                           *IsModified
 604   )
 605 {
 606   UINT64  CurrentPageEntry;
 607   UINT64  NewPageEntry;
 608
 609   CurrentPageEntry = PageEntry->Uint64;
 610   SetSecondLevelPagingEntryAttribute (PageEntry, IoMmuAccess);
 611   FlushPageTableMemory (VtdIndex, (UINTN)PageEntry, sizeof(*PageEntry));
 612   NewPageEntry = PageEntry->Uint64;
 613   if (CurrentPageEntry != NewPageEntry) {
 614     *IsModified = TRUE;
 615     DEBUG ((DEBUG_VERBOSE, "ConvertSecondLevelPageEntryAttribute 0x%lx", CurrentPageEntry));
 616     DEBUG ((DEBUG_VERBOSE, "->0x%lx\n", NewPageEntry));
 617   } else {
 618     *IsModified = FALSE;
 619   }
 620 }
 621
 622 /**
 623   This function returns if there is need to split page entry.
 624
 625   @param[in]  BaseAddress      The base address to be checked.
 626   @param[in]  Length           The length to be checked.
 627   @param[in]  PageAttribute    The page attribute of the page entry.
 628
 629   @retval SplitAttributes on if there is need to split page entry.
 630 **/
 631 PAGE_ATTRIBUTE
 632 NeedSplitPage (
 633   IN  PHYSICAL_ADDRESS                  BaseAddress,
 634   IN  UINT64                            Length,
 635   IN  PAGE_ATTRIBUTE                    PageAttribute
 636   )
 637 {
 638   UINT64                PageEntryLength;
 639
 640   PageEntryLength = PageAttributeToLength (PageAttribute);
 641
 642   if (((BaseAddress & (PageEntryLength - 1)) == 0) && (Length >= PageEntryLength)) {
 643     return PageNone;
 644   }
 645
 646   if (((BaseAddress & PAGING_2M_MASK) != 0) || (Length < SIZE_2MB)) {
 647     return Page4K;
 648   }
 649
 650   return Page2M;
 651 }
 652
 653 /**
 654   This function splits one page entry to small page entries.
 655
 656   @param[in]  VtdIndex         The index used to identify a VTd engine.
 657   @param[in]  PageEntry        The page entry to be splitted.
 658   @param[in]  PageAttribute    The page attribute of the page entry.
 659   @param[in]  SplitAttribute   How to split the page entry.
 660
 661   @retval RETURN_SUCCESS            The page entry is splitted.
 662   @retval RETURN_UNSUPPORTED        The page entry does not support to be splitted.
 663   @retval RETURN_OUT_OF_RESOURCES   No resource to split page entry.
 664 **/
 665 RETURN_STATUS
 666 SplitSecondLevelPage (
 667   IN  UINTN                             VtdIndex,
 668   IN  VTD_SECOND_LEVEL_PAGING_ENTRY     *PageEntry,
 669   IN  PAGE_ATTRIBUTE                    PageAttribute,
 670   IN  PAGE_ATTRIBUTE                    SplitAttribute
 671   )
 672 {
 673   UINT64   BaseAddress;
 674   UINT64   *NewPageEntry;
 675   UINTN    Index;
 676
 677   ASSERT (PageAttribute == Page2M || PageAttribute == Page1G);
 678
 679   if (PageAttribute == Page2M) {
 680     //
 681     // Split 2M to 4K
 682     //
 683     ASSERT (SplitAttribute == Page4K);
 684     if (SplitAttribute == Page4K) {
 685       NewPageEntry = AllocateZeroPages (1);
 686       DEBUG ((DEBUG_VERBOSE, "Split - 0x%x\n", NewPageEntry));
 687       if (NewPageEntry == NULL) {
 688         return RETURN_OUT_OF_RESOURCES;
 689       }
 690       BaseAddress = PageEntry->Uint64 & PAGING_2M_ADDRESS_MASK_64;
 691       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 692         NewPageEntry[Index] = (BaseAddress + SIZE_4KB * Index) | (PageEntry->Uint64 & PAGE_PROGATE_BITS);
 693       }
 694       FlushPageTableMemory (VtdIndex, (UINTN)NewPageEntry, SIZE_4KB);
 695
 696       PageEntry->Uint64 = (UINT64)(UINTN)NewPageEntry;
 697       SetSecondLevelPagingEntryAttribute (PageEntry, EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 698       FlushPageTableMemory (VtdIndex, (UINTN)PageEntry, sizeof(*PageEntry));
 699       return RETURN_SUCCESS;
 700     } else {
 701       return RETURN_UNSUPPORTED;
 702     }
 703   } else if (PageAttribute == Page1G) {
 704     //
 705     // Split 1G to 2M
 706     // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
 707     //
 708     ASSERT (SplitAttribute == Page2M || SplitAttribute == Page4K);
 709     if ((SplitAttribute == Page2M || SplitAttribute == Page4K)) {
 710       NewPageEntry = AllocateZeroPages (1);
 711       DEBUG ((DEBUG_VERBOSE, "Split - 0x%x\n", NewPageEntry));
 712       if (NewPageEntry == NULL) {
 713         return RETURN_OUT_OF_RESOURCES;
 714       }
 715       BaseAddress = PageEntry->Uint64 & PAGING_1G_ADDRESS_MASK_64;
 716       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 717         NewPageEntry[Index] = (BaseAddress + SIZE_2MB * Index) | VTD_PG_PS | (PageEntry->Uint64 & PAGE_PROGATE_BITS);
 718       }
 719       FlushPageTableMemory (VtdIndex, (UINTN)NewPageEntry, SIZE_4KB);
 720
 721       PageEntry->Uint64 = (UINT64)(UINTN)NewPageEntry;
 722       SetSecondLevelPagingEntryAttribute (PageEntry, EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 723       FlushPageTableMemory (VtdIndex, (UINTN)PageEntry, sizeof(*PageEntry));
 724       return RETURN_SUCCESS;
 725     } else {
 726       return RETURN_UNSUPPORTED;
 727     }
 728   } else {
 729     return RETURN_UNSUPPORTED;
 730   }
 731 }
 732
 733 /**
 734   Set VTd attribute for a system memory on second level page entry
 735
 736   @param[in]  VtdIndex                The index used to identify a VTd engine.
 737   @param[in]  DomainIdentifier        The domain ID of the source.
 738   @param[in]  SecondLevelPagingEntry  The second level paging entry in VTd table for the device.
 739   @param[in]  BaseAddress             The base of device memory address to be used as the DMA memory.
 740   @param[in]  Length                  The length of device memory address to be used as the DMA memory.
 741   @param[in]  IoMmuAccess             The IOMMU access.
 742
 743   @retval EFI_SUCCESS            The IoMmuAccess is set for the memory range specified by BaseAddress and Length.
 744   @retval EFI_INVALID_PARAMETER  BaseAddress is not IoMmu Page size aligned.
 745   @retval EFI_INVALID_PARAMETER  Length is not IoMmu Page size aligned.
 746   @retval EFI_INVALID_PARAMETER  Length is 0.
 747   @retval EFI_INVALID_PARAMETER  IoMmuAccess specified an illegal combination of access.
 748   @retval EFI_UNSUPPORTED        The bit mask of IoMmuAccess is not supported by the IOMMU.
 749   @retval EFI_UNSUPPORTED        The IOMMU does not support the memory range specified by BaseAddress and Length.
 750   @retval EFI_OUT_OF_RESOURCES   There are not enough resources available to modify the IOMMU access.
 751   @retval EFI_DEVICE_ERROR       The IOMMU device reported an error while attempting the operation.
 752 **/
 753 EFI_STATUS
 754 SetSecondLevelPagingAttribute (
 755   IN UINTN                         VtdIndex,
 756   IN UINT16                        DomainIdentifier,
 757   IN VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry,
 758   IN UINT64                        BaseAddress,
 759   IN UINT64                        Length,
 760   IN UINT64                        IoMmuAccess
 761   )
 762 {
 763   VTD_SECOND_LEVEL_PAGING_ENTRY  *PageEntry;
 764   PAGE_ATTRIBUTE                 PageAttribute;
 765   UINTN                          PageEntryLength;
 766   PAGE_ATTRIBUTE                 SplitAttribute;
 767   EFI_STATUS                     Status;
 768   BOOLEAN                        IsEntryModified;
 769
 770   DEBUG ((DEBUG_VERBOSE,"SetSecondLevelPagingAttribute (%d) (0x%016lx - 0x%016lx : %x) \n", VtdIndex, BaseAddress, Length, IoMmuAccess));
 771   DEBUG ((DEBUG_VERBOSE,"  SecondLevelPagingEntry Base - 0x%x\n", SecondLevelPagingEntry));
 772
 773   if (BaseAddress != ALIGN_VALUE(BaseAddress, SIZE_4KB)) {
 774     DEBUG ((DEBUG_ERROR, "SetSecondLevelPagingAttribute - Invalid Alignment\n"));
 775     return EFI_UNSUPPORTED;
 776   }
 777   if (Length != ALIGN_VALUE(Length, SIZE_4KB)) {
 778     DEBUG ((DEBUG_ERROR, "SetSecondLevelPagingAttribute - Invalid Alignment\n"));
 779     return EFI_UNSUPPORTED;
 780   }
 781
 782   while (Length != 0) {
 783     PageEntry = GetSecondLevelPageTableEntry (VtdIndex, SecondLevelPagingEntry, BaseAddress, &PageAttribute);
 784     if (PageEntry == NULL) {
 785       DEBUG ((DEBUG_ERROR, "PageEntry - NULL\n"));
 786       return RETURN_UNSUPPORTED;
 787     }
 788     PageEntryLength = PageAttributeToLength (PageAttribute);
 789     SplitAttribute = NeedSplitPage (BaseAddress, Length, PageAttribute);
 790     if (SplitAttribute == PageNone) {
 791       ConvertSecondLevelPageEntryAttribute (VtdIndex, PageEntry, IoMmuAccess, &IsEntryModified);
 792       if (IsEntryModified) {
 793         mVtdUnitInformation[VtdIndex].HasDirtyPages = TRUE;
 794       }
 795       //
 796       // Convert success, move to next
 797       //
 798       BaseAddress += PageEntryLength;
 799       Length -= PageEntryLength;
 800     } else {
 801       Status = SplitSecondLevelPage (VtdIndex, PageEntry, PageAttribute, SplitAttribute);
 802       if (RETURN_ERROR (Status)) {
 803         DEBUG ((DEBUG_ERROR, "SplitSecondLevelPage - %r\n", Status));
 804         return RETURN_UNSUPPORTED;
 805       }
 806       mVtdUnitInformation[VtdIndex].HasDirtyPages = TRUE;
 807       //
 808       // Just split current page
 809       // Convert success in next around
 810       //
 811     }
 812   }
 813
 814   return EFI_SUCCESS;
 815 }
 816
 817 /**
 818   Set VTd attribute for a system memory.
 819
 820   @param[in]  VtdIndex                The index used to identify a VTd engine.
 821   @param[in]  DomainIdentifier        The domain ID of the source.
 822   @param[in]  SecondLevelPagingEntry  The second level paging entry in VTd table for the device.
 823   @param[in]  BaseAddress             The base of device memory address to be used as the DMA memory.
 824   @param[in]  Length                  The length of device memory address to be used as the DMA memory.
 825   @param[in]  IoMmuAccess             The IOMMU access.
 826
 827   @retval EFI_SUCCESS            The IoMmuAccess is set for the memory range specified by BaseAddress and Length.
 828   @retval EFI_INVALID_PARAMETER  BaseAddress is not IoMmu Page size aligned.
 829   @retval EFI_INVALID_PARAMETER  Length is not IoMmu Page size aligned.
 830   @retval EFI_INVALID_PARAMETER  Length is 0.
 831   @retval EFI_INVALID_PARAMETER  IoMmuAccess specified an illegal combination of access.
 832   @retval EFI_UNSUPPORTED        The bit mask of IoMmuAccess is not supported by the IOMMU.
 833   @retval EFI_UNSUPPORTED        The IOMMU does not support the memory range specified by BaseAddress and Length.
 834   @retval EFI_OUT_OF_RESOURCES   There are not enough resources available to modify the IOMMU access.
 835   @retval EFI_DEVICE_ERROR       The IOMMU device reported an error while attempting the operation.
 836 **/
 837 EFI_STATUS
 838 SetPageAttribute (
 839   IN UINTN                         VtdIndex,
 840   IN UINT16                        DomainIdentifier,
 841   IN VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry,
 842   IN UINT64                        BaseAddress,
 843   IN UINT64                        Length,
 844   IN UINT64                        IoMmuAccess
 845   )
 846 {
 847   EFI_STATUS Status;
 848   Status = EFI_NOT_FOUND;
 849   if (SecondLevelPagingEntry != NULL) {
 850     Status = SetSecondLevelPagingAttribute (VtdIndex, DomainIdentifier, SecondLevelPagingEntry, BaseAddress, Length, IoMmuAccess);
 851   }
 852   return Status;
 853 }
 854
 855 /**
 856   Set VTd attribute for a system memory.
 857
 858   @param[in]  Segment           The Segment used to identify a VTd engine.
 859   @param[in]  SourceId          The SourceId used to identify a VTd engine and table entry.
 860   @param[in]  BaseAddress       The base of device memory address to be used as the DMA memory.
 861   @param[in]  Length            The length of device memory address to be used as the DMA memory.
 862   @param[in]  IoMmuAccess       The IOMMU access.
 863
 864   @retval EFI_SUCCESS            The IoMmuAccess is set for the memory range specified by BaseAddress and Length.
 865   @retval EFI_INVALID_PARAMETER  BaseAddress is not IoMmu Page size aligned.
 866   @retval EFI_INVALID_PARAMETER  Length is not IoMmu Page size aligned.
 867   @retval EFI_INVALID_PARAMETER  Length is 0.
 868   @retval EFI_INVALID_PARAMETER  IoMmuAccess specified an illegal combination of access.
 869   @retval EFI_UNSUPPORTED        The bit mask of IoMmuAccess is not supported by the IOMMU.
 870   @retval EFI_UNSUPPORTED        The IOMMU does not support the memory range specified by BaseAddress and Length.
 871   @retval EFI_OUT_OF_RESOURCES   There are not enough resources available to modify the IOMMU access.
 872   @retval EFI_DEVICE_ERROR       The IOMMU device reported an error while attempting the operation.
 873 **/
 874 EFI_STATUS
 875 SetAccessAttribute (
 876   IN UINT16                Segment,
 877   IN VTD_SOURCE_ID         SourceId,
 878   IN UINT64                BaseAddress,
 879   IN UINT64                Length,
 880   IN UINT64                IoMmuAccess
 881   )
 882 {
 883   UINTN                         VtdIndex;
 884   EFI_STATUS                    Status;
 885   VTD_EXT_CONTEXT_ENTRY         *ExtContextEntry;
 886   VTD_CONTEXT_ENTRY             *ContextEntry;
 887   VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry;
 888   UINT64                        Pt;
 889   UINTN                         PciDataIndex;
 890   UINT16                        DomainIdentifier;
 891
 892   SecondLevelPagingEntry = NULL;
 893
 894   DEBUG ((DEBUG_VERBOSE,"SetAccessAttribute (S%04x B%02x D%02x F%02x) (0x%016lx - 0x%08x, %x)\n", Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function, BaseAddress, (UINTN)Length, IoMmuAccess));
 895
 896   VtdIndex = FindVtdIndexByPciDevice (Segment, SourceId, &ExtContextEntry, &ContextEntry);
 897   if (VtdIndex == (UINTN)-1) {
 898     DEBUG ((DEBUG_ERROR,"SetAccessAttribute - Pci device (S%04x B%02x D%02x F%02x) not found!\n", Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 899     return EFI_DEVICE_ERROR;
 900   }
 901
 902   PciDataIndex = GetPciDataIndex (VtdIndex, Segment, SourceId);
 903   mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceData[PciDataIndex].AccessCount++;
 904   //
 905   // DomainId should not be 0.
 906   //
 907   DomainIdentifier = (UINT16)(PciDataIndex + 1);
 908
 909   if (ExtContextEntry != NULL) {
 910     if (ExtContextEntry->Bits.Present == 0) {
 911       SecondLevelPagingEntry = CreateSecondLevelPagingEntry (VtdIndex, 0);
 912       DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry - 0x%x (S%04x B%02x D%02x F%02x) New\n", SecondLevelPagingEntry, Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 913       Pt = (UINT64)RShiftU64 ((UINT64)(UINTN)SecondLevelPagingEntry, 12);
 914
 915       ExtContextEntry->Bits.SecondLevelPageTranslationPointerLo = (UINT32) Pt;
 916       ExtContextEntry->Bits.SecondLevelPageTranslationPointerHi = (UINT32) RShiftU64(Pt, 20);
 917       ExtContextEntry->Bits.DomainIdentifier = DomainIdentifier;
 918       ExtContextEntry->Bits.Present = 1;
 919       FlushPageTableMemory (VtdIndex, (UINTN)ExtContextEntry, sizeof(*ExtContextEntry));
 920       DumpDmarExtContextEntryTable (mVtdUnitInformation[VtdIndex].ExtRootEntryTable);
 921       mVtdUnitInformation[VtdIndex].HasDirtyContext = TRUE;
 922     } else {
 923       SecondLevelPagingEntry = (VOID *)(UINTN)VTD_64BITS_ADDRESS(ExtContextEntry->Bits.SecondLevelPageTranslationPointerLo, ExtContextEntry->Bits.SecondLevelPageTranslationPointerHi);
 924       DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry - 0x%x (S%04x B%02x D%02x F%02x)\n", SecondLevelPagingEntry, Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 925     }
 926   } else if (ContextEntry != NULL) {
 927     if (ContextEntry->Bits.Present == 0) {
 928       SecondLevelPagingEntry = CreateSecondLevelPagingEntry (VtdIndex, 0);
 929       DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry - 0x%x (S%04x B%02x D%02x F%02x) New\n", SecondLevelPagingEntry, Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 930       Pt = (UINT64)RShiftU64 ((UINT64)(UINTN)SecondLevelPagingEntry, 12);
 931
 932       ContextEntry->Bits.SecondLevelPageTranslationPointerLo = (UINT32) Pt;
 933       ContextEntry->Bits.SecondLevelPageTranslationPointerHi = (UINT32) RShiftU64(Pt, 20);
 934       ContextEntry->Bits.DomainIdentifier = DomainIdentifier;
 935       ContextEntry->Bits.Present = 1;
 936       FlushPageTableMemory (VtdIndex, (UINTN)ContextEntry, sizeof(*ContextEntry));
 937       DumpDmarContextEntryTable (mVtdUnitInformation[VtdIndex].RootEntryTable);
 938       mVtdUnitInformation[VtdIndex].HasDirtyContext = TRUE;
 939     } else {
 940       SecondLevelPagingEntry = (VOID *)(UINTN)VTD_64BITS_ADDRESS(ContextEntry->Bits.SecondLevelPageTranslationPointerLo, ContextEntry->Bits.SecondLevelPageTranslationPointerHi);
 941       DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry - 0x%x (S%04x B%02x D%02x F%02x)\n", SecondLevelPagingEntry, Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 942     }
 943   }
 944
 945   //
 946   // Do not update FixedSecondLevelPagingEntry
 947   //
 948   if (SecondLevelPagingEntry != mVtdUnitInformation[VtdIndex].FixedSecondLevelPagingEntry) {
 949     Status = SetPageAttribute (
 950                VtdIndex,
 951                DomainIdentifier,
 952                SecondLevelPagingEntry,
 953                BaseAddress,
 954                Length,
 955                IoMmuAccess
 956                );
 957     if (EFI_ERROR (Status)) {
 958       DEBUG ((DEBUG_ERROR,"SetPageAttribute - %r\n", Status));
 959       return Status;
 960     }
 961   }
 962
 963   InvalidatePageEntry (VtdIndex);
 964
 965   return EFI_SUCCESS;
 966 }
 967
 968 /**
 969   Always enable the VTd page attribute for the device.
 970
 971   @param[in]  Segment           The Segment used to identify a VTd engine.
 972   @param[in]  SourceId          The SourceId used to identify a VTd engine and table entry.
 973
 974   @retval EFI_SUCCESS           The VTd entry is updated to always enable all DMA access for the specific device.
 975 **/
 976 EFI_STATUS
 977 AlwaysEnablePageAttribute (
 978   IN UINT16                  Segment,
 979   IN VTD_SOURCE_ID           SourceId
 980   )
 981 {
 982   UINTN                         VtdIndex;
 983   VTD_EXT_CONTEXT_ENTRY         *ExtContextEntry;
 984   VTD_CONTEXT_ENTRY             *ContextEntry;
 985   VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry;
 986   UINT64                        Pt;
 987
 988   DEBUG ((DEBUG_INFO,"AlwaysEnablePageAttribute (S%04x B%02x D%02x F%02x)\n", Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 989
 990   VtdIndex = FindVtdIndexByPciDevice (Segment, SourceId, &ExtContextEntry, &ContextEntry);
 991   if (VtdIndex == (UINTN)-1) {
 992     DEBUG ((DEBUG_ERROR,"AlwaysEnablePageAttribute - Pci device (S%04x B%02x D%02x F%02x) not found!\n", Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 993     return EFI_DEVICE_ERROR;
 994   }
 995
 996   if (mVtdUnitInformation[VtdIndex].FixedSecondLevelPagingEntry == 0) {
 997     DEBUG((DEBUG_INFO, "CreateSecondLevelPagingEntry - %d\n", VtdIndex));
 998     mVtdUnitInformation[VtdIndex].FixedSecondLevelPagingEntry = CreateSecondLevelPagingEntry (VtdIndex, EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 999   }
1000
1001   SecondLevelPagingEntry = mVtdUnitInformation[VtdIndex].FixedSecondLevelPagingEntry;
1002   Pt = (UINT64)RShiftU64 ((UINT64)(UINTN)SecondLevelPagingEntry, 12);
1003   if (ExtContextEntry != NULL) {
1004     ExtContextEntry->Bits.SecondLevelPageTranslationPointerLo = (UINT32) Pt;
1005     ExtContextEntry->Bits.SecondLevelPageTranslationPointerHi = (UINT32) RShiftU64(Pt, 20);
1006     ExtContextEntry->Bits.DomainIdentifier = ((1 << (UINT8)((UINTN)mVtdUnitInformation[VtdIndex].CapReg.Bits.ND * 2 + 4)) - 1);
1007     ExtContextEntry->Bits.Present = 1;
1008     FlushPageTableMemory (VtdIndex, (UINTN)ExtContextEntry, sizeof(*ExtContextEntry));
1009   } else if (ContextEntry != NULL) {
1010     ContextEntry->Bits.SecondLevelPageTranslationPointerLo = (UINT32) Pt;
1011     ContextEntry->Bits.SecondLevelPageTranslationPointerHi = (UINT32) RShiftU64(Pt, 20);
1012     ContextEntry->Bits.DomainIdentifier = ((1 << (UINT8)((UINTN)mVtdUnitInformation[VtdIndex].CapReg.Bits.ND * 2 + 4)) - 1);
1013     ContextEntry->Bits.Present = 1;
1014     FlushPageTableMemory (VtdIndex, (UINTN)ContextEntry, sizeof(*ContextEntry));
1015   }
1016
1017   return EFI_SUCCESS;
1018 }