IntelSiliconPkg/Feature/VTd/IntelVTdDxe/TranslationTable.c

   1 /** @file
   2
   3   Copyright (c) 2017 - 2018, Intel Corporation. All rights reserved.<BR>
   4   This program and the accompanying materials
   5   are licensed and made available under the terms and conditions of the BSD License
   6   which accompanies this distribution.  The full text of the license may be found at
   7   http://opensource.org/licenses/bsd-license.php.
   8
   9   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  10   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  11
  12 **/
  13
  14 #include "DmaProtection.h"
  15
  16 /**
  17   Create extended context entry.
  18
  19   @param[in]  VtdIndex  The index of the VTd engine.
  20
  21   @retval EFI_SUCCESS          The extended context entry is created.
  22   @retval EFI_OUT_OF_RESOURCE  No enough resource to create extended context entry.
  23 **/
  24 EFI_STATUS
  25 CreateExtContextEntry (
  26   IN UINTN  VtdIndex
  27   );
  28
  29 /**
  30   Allocate zero pages.
  31
  32   @param[in]  Pages the number of pages.
  33
  34   @return the page address.
  35   @retval NULL No resource to allocate pages.
  36 **/
  37 VOID *
  38 EFIAPI
  39 AllocateZeroPages (
  40   IN UINTN  Pages
  41   )
  42 {
  43   VOID *Addr;
  44
  45   Addr = AllocatePages (Pages);
  46   if (Addr == NULL) {
  47     return NULL;
  48   }
  49   ZeroMem (Addr, EFI_PAGES_TO_SIZE(Pages));
  50   return Addr;
  51 }
  52
  53 /**
  54   Set second level paging entry attribute based upon IoMmuAccess.
  55
  56   @param[in]  PtEntry      The paging entry.
  57   @param[in]  IoMmuAccess  The IOMMU access.
  58 **/
  59 VOID
  60 SetSecondLevelPagingEntryAttribute (
  61   IN VTD_SECOND_LEVEL_PAGING_ENTRY  *PtEntry,
  62   IN UINT64                         IoMmuAccess
  63   )
  64 {
  65   PtEntry->Bits.Read  = ((IoMmuAccess & EDKII_IOMMU_ACCESS_READ) != 0);
  66   PtEntry->Bits.Write = ((IoMmuAccess & EDKII_IOMMU_ACCESS_WRITE) != 0);
  67 }
  68
  69 /**
  70   Create context entry.
  71
  72   @param[in]  VtdIndex  The index of the VTd engine.
  73
  74   @retval EFI_SUCCESS          The context entry is created.
  75   @retval EFI_OUT_OF_RESOURCE  No enough resource to create context entry.
  76 **/
  77 EFI_STATUS
  78 CreateContextEntry (
  79   IN UINTN  VtdIndex
  80   )
  81 {
  82   UINTN                  Index;
  83   VOID                   *Buffer;
  84   UINTN                  RootPages;
  85   UINTN                  ContextPages;
  86   VTD_ROOT_ENTRY         *RootEntry;
  87   VTD_CONTEXT_ENTRY      *ContextEntryTable;
  88   VTD_CONTEXT_ENTRY      *ContextEntry;
  89   VTD_SOURCE_ID          *PciSourceId;
  90   VTD_SOURCE_ID          SourceId;
  91   UINTN                  MaxBusNumber;
  92   UINTN                  EntryTablePages;
  93
  94   MaxBusNumber = 0;
  95   for (Index = 0; Index < mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceDataNumber; Index++) {
  96     PciSourceId = &mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceData[Index].PciSourceId;
  97     if (PciSourceId->Bits.Bus > MaxBusNumber) {
  98       MaxBusNumber = PciSourceId->Bits.Bus;
  99     }
 100   }
 101   DEBUG ((DEBUG_INFO,"  MaxBusNumber - 0x%x\n", MaxBusNumber));
 102
 103   RootPages = EFI_SIZE_TO_PAGES (sizeof (VTD_ROOT_ENTRY) * VTD_ROOT_ENTRY_NUMBER);
 104   ContextPages = EFI_SIZE_TO_PAGES (sizeof (VTD_CONTEXT_ENTRY) * VTD_CONTEXT_ENTRY_NUMBER);
 105   EntryTablePages = RootPages + ContextPages * (MaxBusNumber + 1);
 106   Buffer = AllocateZeroPages (EntryTablePages);
 107   if (Buffer == NULL) {
 108     DEBUG ((DEBUG_INFO,"Could not Alloc Root Entry Table.. \n"));
 109     return EFI_OUT_OF_RESOURCES;
 110   }
 111   mVtdUnitInformation[VtdIndex].RootEntryTable = (VTD_ROOT_ENTRY *)Buffer;
 112   Buffer = (UINT8 *)Buffer + EFI_PAGES_TO_SIZE (RootPages);
 113
 114   for (Index = 0; Index < mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceDataNumber; Index++) {
 115     PciSourceId = &mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceData[Index].PciSourceId;
 116
 117     SourceId.Bits.Bus = PciSourceId->Bits.Bus;
 118     SourceId.Bits.Device = PciSourceId->Bits.Device;
 119     SourceId.Bits.Function = PciSourceId->Bits.Function;
 120
 121     RootEntry = &mVtdUnitInformation[VtdIndex].RootEntryTable[SourceId.Index.RootIndex];
 122     if (RootEntry->Bits.Present == 0) {
 123       RootEntry->Bits.ContextTablePointerLo  = (UINT32) RShiftU64 ((UINT64)(UINTN)Buffer, 12);
 124       RootEntry->Bits.ContextTablePointerHi  = (UINT32) RShiftU64 ((UINT64)(UINTN)Buffer, 32);
 125       RootEntry->Bits.Present = 1;
 126       Buffer = (UINT8 *)Buffer + EFI_PAGES_TO_SIZE (ContextPages);
 127     }
 128
 129     ContextEntryTable = (VTD_CONTEXT_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(RootEntry->Bits.ContextTablePointerLo, RootEntry->Bits.ContextTablePointerHi) ;
 130     ContextEntry = &ContextEntryTable[SourceId.Index.ContextIndex];
 131     ContextEntry->Bits.TranslationType = 0;
 132     ContextEntry->Bits.FaultProcessingDisable = 0;
 133     ContextEntry->Bits.Present = 0;
 134
 135     DEBUG ((DEBUG_INFO,"Source: S%04x B%02x D%02x F%02x\n", mVtdUnitInformation[VtdIndex].Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 136
 137     switch (mVtdUnitInformation[VtdIndex].CapReg.Bits.SAGAW) {
 138     case BIT1:
 139       ContextEntry->Bits.AddressWidth = 0x1;
 140       break;
 141     case BIT2:
 142       ContextEntry->Bits.AddressWidth = 0x2;
 143       break;
 144     }
 145   }
 146
 147   FlushPageTableMemory (VtdIndex, (UINTN)mVtdUnitInformation[VtdIndex].RootEntryTable, EFI_PAGES_TO_SIZE(EntryTablePages));
 148
 149   return EFI_SUCCESS;
 150 }
 151
 152 /**
 153   Create second level paging entry table.
 154
 155   @param[in]  VtdIndex                    The index of the VTd engine.
 156   @param[in]  SecondLevelPagingEntry      The second level paging entry.
 157   @param[in]  MemoryBase                  The base of the memory.
 158   @param[in]  MemoryLimit                 The limit of the memory.
 159   @param[in]  IoMmuAccess                 The IOMMU access.
 160
 161   @return The second level paging entry.
 162 **/
 163 VTD_SECOND_LEVEL_PAGING_ENTRY *
 164 CreateSecondLevelPagingEntryTable (
 165   IN UINTN                         VtdIndex,
 166   IN VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry,
 167   IN UINT64                        MemoryBase,
 168   IN UINT64                        MemoryLimit,
 169   IN UINT64                        IoMmuAccess
 170   )
 171 {
 172   UINTN                          Index4;
 173   UINTN                          Index3;
 174   UINTN                          Index2;
 175   UINTN                          Lvl4Start;
 176   UINTN                          Lvl4End;
 177   UINTN                          Lvl3Start;
 178   UINTN                          Lvl3End;
 179   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl4PtEntry;
 180   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl3PtEntry;
 181   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl2PtEntry;
 182   UINT64                         BaseAddress;
 183   UINT64                         EndAddress;
 184
 185   if (MemoryLimit == 0) {
 186     return EFI_SUCCESS;
 187   }
 188
 189   BaseAddress = ALIGN_VALUE_LOW(MemoryBase, SIZE_2MB);
 190   EndAddress = ALIGN_VALUE_UP(MemoryLimit, SIZE_2MB);
 191   DEBUG ((DEBUG_INFO,"CreateSecondLevelPagingEntryTable: BaseAddress - 0x%016lx, EndAddress - 0x%016lx\n", BaseAddress, EndAddress));
 192
 193   if (SecondLevelPagingEntry == NULL) {
 194     SecondLevelPagingEntry = AllocateZeroPages (1);
 195     if (SecondLevelPagingEntry == NULL) {
 196       DEBUG ((DEBUG_ERROR,"Could not Alloc LVL4 PT. \n"));
 197       return NULL;
 198     }
 199     FlushPageTableMemory (VtdIndex, (UINTN)SecondLevelPagingEntry, EFI_PAGES_TO_SIZE(1));
 200   }
 201
 202   //
 203   // If no access is needed, just create not present entry.
 204   //
 205   if (IoMmuAccess == 0) {
 206     return SecondLevelPagingEntry;
 207   }
 208
 209   Lvl4Start = RShiftU64 (BaseAddress, 39) & 0x1FF;
 210   Lvl4End = RShiftU64 (EndAddress - 1, 39) & 0x1FF;
 211
 212   DEBUG ((DEBUG_INFO,"  Lvl4Start - 0x%x, Lvl4End - 0x%x\n", Lvl4Start, Lvl4End));
 213
 214   Lvl4PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)SecondLevelPagingEntry;
 215   for (Index4 = Lvl4Start; Index4 <= Lvl4End; Index4++) {
 216     if (Lvl4PtEntry[Index4].Uint64 == 0) {
 217       Lvl4PtEntry[Index4].Uint64 = (UINT64)(UINTN)AllocateZeroPages (1);
 218       if (Lvl4PtEntry[Index4].Uint64 == 0) {
 219         DEBUG ((DEBUG_ERROR,"!!!!!! ALLOCATE LVL4 PAGE FAIL (0x%x)!!!!!!\n", Index4));
 220         ASSERT(FALSE);
 221         return NULL;
 222       }
 223       FlushPageTableMemory (VtdIndex, (UINTN)Lvl4PtEntry[Index4].Uint64, SIZE_4KB);
 224       SetSecondLevelPagingEntryAttribute (&Lvl4PtEntry[Index4], EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 225     }
 226
 227     Lvl3Start = RShiftU64 (BaseAddress, 30) & 0x1FF;
 228     if (ALIGN_VALUE_LOW(BaseAddress + SIZE_1GB, SIZE_1GB) <= EndAddress) {
 229       Lvl3End = SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY) - 1;
 230     } else {
 231       Lvl3End = RShiftU64 (EndAddress - 1, 30) & 0x1FF;
 232     }
 233     DEBUG ((DEBUG_INFO,"  Lvl4(0x%x): Lvl3Start - 0x%x, Lvl3End - 0x%x\n", Index4, Lvl3Start, Lvl3End));
 234
 235     Lvl3PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl4PtEntry[Index4].Bits.AddressLo, Lvl4PtEntry[Index4].Bits.AddressHi);
 236     for (Index3 = Lvl3Start; Index3 <= Lvl3End; Index3++) {
 237       if (Lvl3PtEntry[Index3].Uint64 == 0) {
 238         Lvl3PtEntry[Index3].Uint64 = (UINT64)(UINTN)AllocateZeroPages (1);
 239         if (Lvl3PtEntry[Index3].Uint64 == 0) {
 240           DEBUG ((DEBUG_ERROR,"!!!!!! ALLOCATE LVL3 PAGE FAIL (0x%x, 0x%x)!!!!!!\n", Index4, Index3));
 241           ASSERT(FALSE);
 242           return NULL;
 243         }
 244         FlushPageTableMemory (VtdIndex, (UINTN)Lvl3PtEntry[Index3].Uint64, SIZE_4KB);
 245         SetSecondLevelPagingEntryAttribute (&Lvl3PtEntry[Index3], EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 246       }
 247
 248       Lvl2PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl3PtEntry[Index3].Bits.AddressLo, Lvl3PtEntry[Index3].Bits.AddressHi);
 249       for (Index2 = 0; Index2 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index2++) {
 250         Lvl2PtEntry[Index2].Uint64 = BaseAddress;
 251         SetSecondLevelPagingEntryAttribute (&Lvl2PtEntry[Index2], IoMmuAccess);
 252         Lvl2PtEntry[Index2].Bits.PageSize = 1;
 253         BaseAddress += SIZE_2MB;
 254         if (BaseAddress >= MemoryLimit) {
 255           break;
 256         }
 257       }
 258       FlushPageTableMemory (VtdIndex, (UINTN)Lvl2PtEntry, SIZE_4KB);
 259       if (BaseAddress >= MemoryLimit) {
 260         break;
 261       }
 262     }
 263     FlushPageTableMemory (VtdIndex, (UINTN)&Lvl3PtEntry[Lvl3Start], (UINTN)&Lvl3PtEntry[Lvl3End + 1] - (UINTN)&Lvl3PtEntry[Lvl3Start]);
 264     if (BaseAddress >= MemoryLimit) {
 265       break;
 266     }
 267   }
 268   FlushPageTableMemory (VtdIndex, (UINTN)&Lvl4PtEntry[Lvl4Start], (UINTN)&Lvl4PtEntry[Lvl4End + 1] - (UINTN)&Lvl4PtEntry[Lvl4Start]);
 269
 270   return SecondLevelPagingEntry;
 271 }
 272
 273 /**
 274   Create second level paging entry.
 275
 276   @param[in]  VtdIndex                    The index of the VTd engine.
 277   @param[in]  IoMmuAccess                 The IOMMU access.
 278
 279   @return The second level paging entry.
 280 **/
 281 VTD_SECOND_LEVEL_PAGING_ENTRY *
 282 CreateSecondLevelPagingEntry (
 283   IN UINTN   VtdIndex,
 284   IN UINT64  IoMmuAccess
 285   )
 286 {
 287   VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry;
 288
 289   SecondLevelPagingEntry = NULL;
 290   SecondLevelPagingEntry = CreateSecondLevelPagingEntryTable (VtdIndex, SecondLevelPagingEntry, 0, mBelow4GMemoryLimit, IoMmuAccess);
 291   if (SecondLevelPagingEntry == NULL) {
 292     return NULL;
 293   }
 294
 295   if (mAbove4GMemoryLimit != 0) {
 296     ASSERT (mAbove4GMemoryLimit > BASE_4GB);
 297     SecondLevelPagingEntry = CreateSecondLevelPagingEntryTable (VtdIndex, SecondLevelPagingEntry, SIZE_4GB, mAbove4GMemoryLimit, IoMmuAccess);
 298     if (SecondLevelPagingEntry == NULL) {
 299       return NULL;
 300     }
 301   }
 302
 303   return SecondLevelPagingEntry;
 304 }
 305
 306 /**
 307   Setup VTd translation table.
 308
 309   @retval EFI_SUCCESS           Setup translation table successfully.
 310   @retval EFI_OUT_OF_RESOURCE   Setup translation table fail.
 311 **/
 312 EFI_STATUS
 313 SetupTranslationTable (
 314   VOID
 315   )
 316 {
 317   EFI_STATUS      Status;
 318   UINTN           Index;
 319
 320   for (Index = 0; Index < mVtdUnitNumber; Index++) {
 321     DEBUG((DEBUG_INFO, "CreateContextEntry - %d\n", Index));
 322     if (mVtdUnitInformation[Index].ECapReg.Bits.ECS) {
 323       Status = CreateExtContextEntry (Index);
 324     } else {
 325       Status = CreateContextEntry (Index);
 326     }
 327     if (EFI_ERROR (Status)) {
 328       return Status;
 329     }
 330   }
 331
 332   return EFI_SUCCESS;
 333 }
 334
 335 /**
 336   Dump DMAR context entry table.
 337
 338   @param[in]  RootEntry DMAR root entry.
 339 **/
 340 VOID
 341 DumpDmarContextEntryTable (
 342   IN VTD_ROOT_ENTRY *RootEntry
 343   )
 344 {
 345   UINTN                 Index;
 346   UINTN                 Index2;
 347   VTD_CONTEXT_ENTRY     *ContextEntry;
 348
 349   DEBUG ((DEBUG_INFO,"=========================\n"));
 350   DEBUG ((DEBUG_INFO,"DMAR Context Entry Table:\n"));
 351
 352   DEBUG ((DEBUG_INFO,"RootEntry Address - 0x%x\n", RootEntry));
 353
 354   for (Index = 0; Index < VTD_ROOT_ENTRY_NUMBER; Index++) {
 355     if ((RootEntry[Index].Uint128.Uint64Lo != 0) || (RootEntry[Index].Uint128.Uint64Hi != 0)) {
 356       DEBUG ((DEBUG_INFO,"  RootEntry(0x%02x) B%02x - 0x%016lx %016lx\n",
 357         Index, Index, RootEntry[Index].Uint128.Uint64Hi, RootEntry[Index].Uint128.Uint64Lo));
 358     }
 359     if (RootEntry[Index].Bits.Present == 0) {
 360       continue;
 361     }
 362     ContextEntry = (VTD_CONTEXT_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(RootEntry[Index].Bits.ContextTablePointerLo, RootEntry[Index].Bits.ContextTablePointerHi);
 363     for (Index2 = 0; Index2 < VTD_CONTEXT_ENTRY_NUMBER; Index2++) {
 364       if ((ContextEntry[Index2].Uint128.Uint64Lo != 0) || (ContextEntry[Index2].Uint128.Uint64Hi != 0)) {
 365         DEBUG ((DEBUG_INFO,"    ContextEntry(0x%02x) D%02xF%02x - 0x%016lx %016lx\n",
 366           Index2, Index2 >> 3, Index2 & 0x7, ContextEntry[Index2].Uint128.Uint64Hi, ContextEntry[Index2].Uint128.Uint64Lo));
 367       }
 368       if (ContextEntry[Index2].Bits.Present == 0) {
 369         continue;
 370       }
 371       DumpSecondLevelPagingEntry ((VOID *)(UINTN)VTD_64BITS_ADDRESS(ContextEntry[Index2].Bits.SecondLevelPageTranslationPointerLo, ContextEntry[Index2].Bits.SecondLevelPageTranslationPointerHi));
 372     }
 373   }
 374   DEBUG ((DEBUG_INFO,"=========================\n"));
 375 }
 376
 377 /**
 378   Dump DMAR second level paging entry.
 379
 380   @param[in]  SecondLevelPagingEntry The second level paging entry.
 381 **/
 382 VOID
 383 DumpSecondLevelPagingEntry (
 384   IN VOID *SecondLevelPagingEntry
 385   )
 386 {
 387   UINTN                          Index4;
 388   UINTN                          Index3;
 389   UINTN                          Index2;
 390   UINTN                          Index1;
 391   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl4PtEntry;
 392   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl3PtEntry;
 393   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl2PtEntry;
 394   VTD_SECOND_LEVEL_PAGING_ENTRY  *Lvl1PtEntry;
 395
 396   DEBUG ((DEBUG_VERBOSE,"================\n"));
 397   DEBUG ((DEBUG_VERBOSE,"DMAR Second Level Page Table:\n"));
 398
 399   DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry Base - 0x%x\n", SecondLevelPagingEntry));
 400   Lvl4PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)SecondLevelPagingEntry;
 401   for (Index4 = 0; Index4 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index4++) {
 402     if (Lvl4PtEntry[Index4].Uint64 != 0) {
 403       DEBUG ((DEBUG_VERBOSE,"  Lvl4Pt Entry(0x%03x) - 0x%016lx\n", Index4, Lvl4PtEntry[Index4].Uint64));
 404     }
 405     if (Lvl4PtEntry[Index4].Uint64 == 0) {
 406       continue;
 407     }
 408     Lvl3PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl4PtEntry[Index4].Bits.AddressLo, Lvl4PtEntry[Index4].Bits.AddressHi);
 409     for (Index3 = 0; Index3 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index3++) {
 410       if (Lvl3PtEntry[Index3].Uint64 != 0) {
 411         DEBUG ((DEBUG_VERBOSE,"    Lvl3Pt Entry(0x%03x) - 0x%016lx\n", Index3, Lvl3PtEntry[Index3].Uint64));
 412       }
 413       if (Lvl3PtEntry[Index3].Uint64 == 0) {
 414         continue;
 415       }
 416
 417       Lvl2PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl3PtEntry[Index3].Bits.AddressLo, Lvl3PtEntry[Index3].Bits.AddressHi);
 418       for (Index2 = 0; Index2 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index2++) {
 419         if (Lvl2PtEntry[Index2].Uint64 != 0) {
 420           DEBUG ((DEBUG_VERBOSE,"      Lvl2Pt Entry(0x%03x) - 0x%016lx\n", Index2, Lvl2PtEntry[Index2].Uint64));
 421         }
 422         if (Lvl2PtEntry[Index2].Uint64 == 0) {
 423           continue;
 424         }
 425         if (Lvl2PtEntry[Index2].Bits.PageSize == 0) {
 426           Lvl1PtEntry = (VTD_SECOND_LEVEL_PAGING_ENTRY *)(UINTN)VTD_64BITS_ADDRESS(Lvl2PtEntry[Index2].Bits.AddressLo, Lvl2PtEntry[Index2].Bits.AddressHi);
 427           for (Index1 = 0; Index1 < SIZE_4KB/sizeof(VTD_SECOND_LEVEL_PAGING_ENTRY); Index1++) {
 428             if (Lvl1PtEntry[Index1].Uint64 != 0) {
 429               DEBUG ((DEBUG_VERBOSE,"        Lvl1Pt Entry(0x%03x) - 0x%016lx\n", Index1, Lvl1PtEntry[Index1].Uint64));
 430             }
 431           }
 432         }
 433       }
 434     }
 435   }
 436   DEBUG ((DEBUG_VERBOSE,"================\n"));
 437 }
 438
 439 /**
 440   Invalid page entry.
 441
 442   @param VtdIndex  The VTd engine index.
 443 **/
 444 VOID
 445 InvalidatePageEntry (
 446   IN UINTN                 VtdIndex
 447   )
 448 {
 449   if (mVtdUnitInformation[VtdIndex].HasDirtyContext || mVtdUnitInformation[VtdIndex].HasDirtyPages) {
 450     InvalidateVtdIOTLBGlobal (VtdIndex);
 451   }
 452   mVtdUnitInformation[VtdIndex].HasDirtyContext = FALSE;
 453   mVtdUnitInformation[VtdIndex].HasDirtyPages = FALSE;
 454 }
 455
 456 #define VTD_PG_R                   BIT0
 457 #define VTD_PG_W                   BIT1
 458 #define VTD_PG_X                   BIT2
 459 #define VTD_PG_EMT                 (BIT3 | BIT4 | BIT5)
 460 #define VTD_PG_TM                  (BIT62)
 461
 462 #define VTD_PG_PS                  BIT7
 463
 464 #define PAGE_PROGATE_BITS          (VTD_PG_TM | VTD_PG_EMT | VTD_PG_W | VTD_PG_R)
 465
 466 #define PAGING_4K_MASK  0xFFF
 467 #define PAGING_2M_MASK  0x1FFFFF
 468 #define PAGING_1G_MASK  0x3FFFFFFF
 469
 470 #define PAGING_VTD_INDEX_MASK     0x1FF
 471
 472 #define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull
 473 #define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull
 474 #define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull
 475
 476 typedef enum {
 477   PageNone,
 478   Page4K,
 479   Page2M,
 480   Page1G,
 481 } PAGE_ATTRIBUTE;
 482
 483 typedef struct {
 484   PAGE_ATTRIBUTE   Attribute;
 485   UINT64           Length;
 486   UINT64           AddressMask;
 487 } PAGE_ATTRIBUTE_TABLE;
 488
 489 PAGE_ATTRIBUTE_TABLE mPageAttributeTable[] = {
 490   {Page4K,  SIZE_4KB, PAGING_4K_ADDRESS_MASK_64},
 491   {Page2M,  SIZE_2MB, PAGING_2M_ADDRESS_MASK_64},
 492   {Page1G,  SIZE_1GB, PAGING_1G_ADDRESS_MASK_64},
 493 };
 494
 495 /**
 496   Return length according to page attributes.
 497
 498   @param[in]  PageAttributes   The page attribute of the page entry.
 499
 500   @return The length of page entry.
 501 **/
 502 UINTN
 503 PageAttributeToLength (
 504   IN PAGE_ATTRIBUTE  PageAttribute
 505   )
 506 {
 507   UINTN  Index;
 508   for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {
 509     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 510       return (UINTN)mPageAttributeTable[Index].Length;
 511     }
 512   }
 513   return 0;
 514 }
 515
 516 /**
 517   Return page table entry to match the address.
 518
 519   @param[in]   VtdIndex                 The index used to identify a VTd engine.
 520   @param[in]   SecondLevelPagingEntry   The second level paging entry in VTd table for the device.
 521   @param[in]   Address                  The address to be checked.
 522   @param[out]  PageAttributes           The page attribute of the page entry.
 523
 524   @return The page entry.
 525 **/
 526 VOID *
 527 GetSecondLevelPageTableEntry (
 528   IN  UINTN                         VtdIndex,
 529   IN  VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry,
 530   IN  PHYSICAL_ADDRESS              Address,
 531   OUT PAGE_ATTRIBUTE                *PageAttribute
 532   )
 533 {
 534   UINTN                 Index1;
 535   UINTN                 Index2;
 536   UINTN                 Index3;
 537   UINTN                 Index4;
 538   UINT64                *L1PageTable;
 539   UINT64                *L2PageTable;
 540   UINT64                *L3PageTable;
 541   UINT64                *L4PageTable;
 542
 543   Index4 = ((UINTN)RShiftU64 (Address, 39)) & PAGING_VTD_INDEX_MASK;
 544   Index3 = ((UINTN)Address >> 30) & PAGING_VTD_INDEX_MASK;
 545   Index2 = ((UINTN)Address >> 21) & PAGING_VTD_INDEX_MASK;
 546   Index1 = ((UINTN)Address >> 12) & PAGING_VTD_INDEX_MASK;
 547
 548   L4PageTable = (UINT64 *)SecondLevelPagingEntry;
 549   if (L4PageTable[Index4] == 0) {
 550     L4PageTable[Index4] = (UINT64)(UINTN)AllocateZeroPages (1);
 551     if (L4PageTable[Index4] == 0) {
 552       DEBUG ((DEBUG_ERROR,"!!!!!! ALLOCATE LVL4 PAGE FAIL (0x%x)!!!!!!\n", Index4));
 553       ASSERT(FALSE);
 554       *PageAttribute = PageNone;
 555       return NULL;
 556     }
 557     FlushPageTableMemory (VtdIndex, (UINTN)L4PageTable[Index4], SIZE_4KB);
 558     SetSecondLevelPagingEntryAttribute ((VTD_SECOND_LEVEL_PAGING_ENTRY *)&L4PageTable[Index4], EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 559     FlushPageTableMemory (VtdIndex, (UINTN)&L4PageTable[Index4], sizeof(L4PageTable[Index4]));
 560   }
 561
 562   L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & PAGING_4K_ADDRESS_MASK_64);
 563   if (L3PageTable[Index3] == 0) {
 564     L3PageTable[Index3] = (UINT64)(UINTN)AllocateZeroPages (1);
 565     if (L3PageTable[Index3] == 0) {
 566       DEBUG ((DEBUG_ERROR,"!!!!!! ALLOCATE LVL3 PAGE FAIL (0x%x, 0x%x)!!!!!!\n", Index4, Index3));
 567       ASSERT(FALSE);
 568       *PageAttribute = PageNone;
 569       return NULL;
 570     }
 571     FlushPageTableMemory (VtdIndex, (UINTN)L3PageTable[Index3], SIZE_4KB);
 572     SetSecondLevelPagingEntryAttribute ((VTD_SECOND_LEVEL_PAGING_ENTRY *)&L3PageTable[Index3], EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 573     FlushPageTableMemory (VtdIndex, (UINTN)&L3PageTable[Index3], sizeof(L3PageTable[Index3]));
 574   }
 575   if ((L3PageTable[Index3] & VTD_PG_PS) != 0) {
 576     // 1G
 577     *PageAttribute = Page1G;
 578     return &L3PageTable[Index3];
 579   }
 580
 581   L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & PAGING_4K_ADDRESS_MASK_64);
 582   if (L2PageTable[Index2] == 0) {
 583     L2PageTable[Index2] = Address & PAGING_2M_ADDRESS_MASK_64;
 584     SetSecondLevelPagingEntryAttribute ((VTD_SECOND_LEVEL_PAGING_ENTRY *)&L2PageTable[Index2], 0);
 585     L2PageTable[Index2] |= VTD_PG_PS;
 586     FlushPageTableMemory (VtdIndex, (UINTN)&L2PageTable[Index2], sizeof(L2PageTable[Index2]));
 587   }
 588   if ((L2PageTable[Index2] & VTD_PG_PS) != 0) {
 589     // 2M
 590     *PageAttribute = Page2M;
 591     return &L2PageTable[Index2];
 592   }
 593
 594   // 4k
 595   L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & PAGING_4K_ADDRESS_MASK_64);
 596   if ((L1PageTable[Index1] == 0) && (Address != 0)) {
 597     *PageAttribute = PageNone;
 598     return NULL;
 599   }
 600   *PageAttribute = Page4K;
 601   return &L1PageTable[Index1];
 602 }
 603
 604 /**
 605   Modify memory attributes of page entry.
 606
 607   @param[in]   VtdIndex         The index used to identify a VTd engine.
 608   @param[in]   PageEntry        The page entry.
 609   @param[in]   IoMmuAccess      The IOMMU access.
 610   @param[out]  IsModified       TRUE means page table modified. FALSE means page table not modified.
 611 **/
 612 VOID
 613 ConvertSecondLevelPageEntryAttribute (
 614   IN  UINTN                             VtdIndex,
 615   IN  VTD_SECOND_LEVEL_PAGING_ENTRY     *PageEntry,
 616   IN  UINT64                            IoMmuAccess,
 617   OUT BOOLEAN                           *IsModified
 618   )
 619 {
 620   UINT64  CurrentPageEntry;
 621   UINT64  NewPageEntry;
 622
 623   CurrentPageEntry = PageEntry->Uint64;
 624   SetSecondLevelPagingEntryAttribute (PageEntry, IoMmuAccess);
 625   FlushPageTableMemory (VtdIndex, (UINTN)PageEntry, sizeof(*PageEntry));
 626   NewPageEntry = PageEntry->Uint64;
 627   if (CurrentPageEntry != NewPageEntry) {
 628     *IsModified = TRUE;
 629     DEBUG ((DEBUG_VERBOSE, "ConvertSecondLevelPageEntryAttribute 0x%lx", CurrentPageEntry));
 630     DEBUG ((DEBUG_VERBOSE, "->0x%lx\n", NewPageEntry));
 631   } else {
 632     *IsModified = FALSE;
 633   }
 634 }
 635
 636 /**
 637   This function returns if there is need to split page entry.
 638
 639   @param[in]  BaseAddress      The base address to be checked.
 640   @param[in]  Length           The length to be checked.
 641   @param[in]  PageAttribute    The page attribute of the page entry.
 642
 643   @retval SplitAttributes on if there is need to split page entry.
 644 **/
 645 PAGE_ATTRIBUTE
 646 NeedSplitPage (
 647   IN  PHYSICAL_ADDRESS                  BaseAddress,
 648   IN  UINT64                            Length,
 649   IN  PAGE_ATTRIBUTE                    PageAttribute
 650   )
 651 {
 652   UINT64                PageEntryLength;
 653
 654   PageEntryLength = PageAttributeToLength (PageAttribute);
 655
 656   if (((BaseAddress & (PageEntryLength - 1)) == 0) && (Length >= PageEntryLength)) {
 657     return PageNone;
 658   }
 659
 660   if (((BaseAddress & PAGING_2M_MASK) != 0) || (Length < SIZE_2MB)) {
 661     return Page4K;
 662   }
 663
 664   return Page2M;
 665 }
 666
 667 /**
 668   This function splits one page entry to small page entries.
 669
 670   @param[in]  VtdIndex         The index used to identify a VTd engine.
 671   @param[in]  PageEntry        The page entry to be splitted.
 672   @param[in]  PageAttribute    The page attribute of the page entry.
 673   @param[in]  SplitAttribute   How to split the page entry.
 674
 675   @retval RETURN_SUCCESS            The page entry is splitted.
 676   @retval RETURN_UNSUPPORTED        The page entry does not support to be splitted.
 677   @retval RETURN_OUT_OF_RESOURCES   No resource to split page entry.
 678 **/
 679 RETURN_STATUS
 680 SplitSecondLevelPage (
 681   IN  UINTN                             VtdIndex,
 682   IN  VTD_SECOND_LEVEL_PAGING_ENTRY     *PageEntry,
 683   IN  PAGE_ATTRIBUTE                    PageAttribute,
 684   IN  PAGE_ATTRIBUTE                    SplitAttribute
 685   )
 686 {
 687   UINT64   BaseAddress;
 688   UINT64   *NewPageEntry;
 689   UINTN    Index;
 690
 691   ASSERT (PageAttribute == Page2M || PageAttribute == Page1G);
 692
 693   if (PageAttribute == Page2M) {
 694     //
 695     // Split 2M to 4K
 696     //
 697     ASSERT (SplitAttribute == Page4K);
 698     if (SplitAttribute == Page4K) {
 699       NewPageEntry = AllocateZeroPages (1);
 700       DEBUG ((DEBUG_VERBOSE, "Split - 0x%x\n", NewPageEntry));
 701       if (NewPageEntry == NULL) {
 702         return RETURN_OUT_OF_RESOURCES;
 703       }
 704       BaseAddress = PageEntry->Uint64 & PAGING_2M_ADDRESS_MASK_64;
 705       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 706         NewPageEntry[Index] = (BaseAddress + SIZE_4KB * Index) | (PageEntry->Uint64 & PAGE_PROGATE_BITS);
 707       }
 708       FlushPageTableMemory (VtdIndex, (UINTN)NewPageEntry, SIZE_4KB);
 709
 710       PageEntry->Uint64 = (UINT64)(UINTN)NewPageEntry;
 711       SetSecondLevelPagingEntryAttribute (PageEntry, EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 712       FlushPageTableMemory (VtdIndex, (UINTN)PageEntry, sizeof(*PageEntry));
 713       return RETURN_SUCCESS;
 714     } else {
 715       return RETURN_UNSUPPORTED;
 716     }
 717   } else if (PageAttribute == Page1G) {
 718     //
 719     // Split 1G to 2M
 720     // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
 721     //
 722     ASSERT (SplitAttribute == Page2M || SplitAttribute == Page4K);
 723     if ((SplitAttribute == Page2M || SplitAttribute == Page4K)) {
 724       NewPageEntry = AllocateZeroPages (1);
 725       DEBUG ((DEBUG_VERBOSE, "Split - 0x%x\n", NewPageEntry));
 726       if (NewPageEntry == NULL) {
 727         return RETURN_OUT_OF_RESOURCES;
 728       }
 729       BaseAddress = PageEntry->Uint64 & PAGING_1G_ADDRESS_MASK_64;
 730       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 731         NewPageEntry[Index] = (BaseAddress + SIZE_2MB * Index) | VTD_PG_PS | (PageEntry->Uint64 & PAGE_PROGATE_BITS);
 732       }
 733       FlushPageTableMemory (VtdIndex, (UINTN)NewPageEntry, SIZE_4KB);
 734
 735       PageEntry->Uint64 = (UINT64)(UINTN)NewPageEntry;
 736       SetSecondLevelPagingEntryAttribute (PageEntry, EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
 737       FlushPageTableMemory (VtdIndex, (UINTN)PageEntry, sizeof(*PageEntry));
 738       return RETURN_SUCCESS;
 739     } else {
 740       return RETURN_UNSUPPORTED;
 741     }
 742   } else {
 743     return RETURN_UNSUPPORTED;
 744   }
 745 }
 746
 747 /**
 748   Set VTd attribute for a system memory on second level page entry
 749
 750   @param[in]  VtdIndex                The index used to identify a VTd engine.
 751   @param[in]  DomainIdentifier        The domain ID of the source.
 752   @param[in]  SecondLevelPagingEntry  The second level paging entry in VTd table for the device.
 753   @param[in]  BaseAddress             The base of device memory address to be used as the DMA memory.
 754   @param[in]  Length                  The length of device memory address to be used as the DMA memory.
 755   @param[in]  IoMmuAccess             The IOMMU access.
 756
 757   @retval EFI_SUCCESS            The IoMmuAccess is set for the memory range specified by BaseAddress and Length.
 758   @retval EFI_INVALID_PARAMETER  BaseAddress is not IoMmu Page size aligned.
 759   @retval EFI_INVALID_PARAMETER  Length is not IoMmu Page size aligned.
 760   @retval EFI_INVALID_PARAMETER  Length is 0.
 761   @retval EFI_INVALID_PARAMETER  IoMmuAccess specified an illegal combination of access.
 762   @retval EFI_UNSUPPORTED        The bit mask of IoMmuAccess is not supported by the IOMMU.
 763   @retval EFI_UNSUPPORTED        The IOMMU does not support the memory range specified by BaseAddress and Length.
 764   @retval EFI_OUT_OF_RESOURCES   There are not enough resources available to modify the IOMMU access.
 765   @retval EFI_DEVICE_ERROR       The IOMMU device reported an error while attempting the operation.
 766 **/
 767 EFI_STATUS
 768 SetSecondLevelPagingAttribute (
 769   IN UINTN                         VtdIndex,
 770   IN UINT16                        DomainIdentifier,
 771   IN VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry,
 772   IN UINT64                        BaseAddress,
 773   IN UINT64                        Length,
 774   IN UINT64                        IoMmuAccess
 775   )
 776 {
 777   VTD_SECOND_LEVEL_PAGING_ENTRY  *PageEntry;
 778   PAGE_ATTRIBUTE                 PageAttribute;
 779   UINTN                          PageEntryLength;
 780   PAGE_ATTRIBUTE                 SplitAttribute;
 781   EFI_STATUS                     Status;
 782   BOOLEAN                        IsEntryModified;
 783
 784   DEBUG ((DEBUG_VERBOSE,"SetSecondLevelPagingAttribute (%d) (0x%016lx - 0x%016lx : %x) \n", VtdIndex, BaseAddress, Length, IoMmuAccess));
 785   DEBUG ((DEBUG_VERBOSE,"  SecondLevelPagingEntry Base - 0x%x\n", SecondLevelPagingEntry));
 786
 787   if (BaseAddress != ALIGN_VALUE(BaseAddress, SIZE_4KB)) {
 788     DEBUG ((DEBUG_ERROR, "SetSecondLevelPagingAttribute - Invalid Alignment\n"));
 789     return EFI_UNSUPPORTED;
 790   }
 791   if (Length != ALIGN_VALUE(Length, SIZE_4KB)) {
 792     DEBUG ((DEBUG_ERROR, "SetSecondLevelPagingAttribute - Invalid Alignment\n"));
 793     return EFI_UNSUPPORTED;
 794   }
 795
 796   while (Length != 0) {
 797     PageEntry = GetSecondLevelPageTableEntry (VtdIndex, SecondLevelPagingEntry, BaseAddress, &PageAttribute);
 798     if (PageEntry == NULL) {
 799       DEBUG ((DEBUG_ERROR, "PageEntry - NULL\n"));
 800       return RETURN_UNSUPPORTED;
 801     }
 802     PageEntryLength = PageAttributeToLength (PageAttribute);
 803     SplitAttribute = NeedSplitPage (BaseAddress, Length, PageAttribute);
 804     if (SplitAttribute == PageNone) {
 805       ConvertSecondLevelPageEntryAttribute (VtdIndex, PageEntry, IoMmuAccess, &IsEntryModified);
 806       if (IsEntryModified) {
 807         mVtdUnitInformation[VtdIndex].HasDirtyPages = TRUE;
 808       }
 809       //
 810       // Convert success, move to next
 811       //
 812       BaseAddress += PageEntryLength;
 813       Length -= PageEntryLength;
 814     } else {
 815       Status = SplitSecondLevelPage (VtdIndex, PageEntry, PageAttribute, SplitAttribute);
 816       if (RETURN_ERROR (Status)) {
 817         DEBUG ((DEBUG_ERROR, "SplitSecondLevelPage - %r\n", Status));
 818         return RETURN_UNSUPPORTED;
 819       }
 820       mVtdUnitInformation[VtdIndex].HasDirtyPages = TRUE;
 821       //
 822       // Just split current page
 823       // Convert success in next around
 824       //
 825     }
 826   }
 827
 828   return EFI_SUCCESS;
 829 }
 830
 831 /**
 832   Set VTd attribute for a system memory.
 833
 834   @param[in]  VtdIndex                The index used to identify a VTd engine.
 835   @param[in]  DomainIdentifier        The domain ID of the source.
 836   @param[in]  SecondLevelPagingEntry  The second level paging entry in VTd table for the device.
 837   @param[in]  BaseAddress             The base of device memory address to be used as the DMA memory.
 838   @param[in]  Length                  The length of device memory address to be used as the DMA memory.
 839   @param[in]  IoMmuAccess             The IOMMU access.
 840
 841   @retval EFI_SUCCESS            The IoMmuAccess is set for the memory range specified by BaseAddress and Length.
 842   @retval EFI_INVALID_PARAMETER  BaseAddress is not IoMmu Page size aligned.
 843   @retval EFI_INVALID_PARAMETER  Length is not IoMmu Page size aligned.
 844   @retval EFI_INVALID_PARAMETER  Length is 0.
 845   @retval EFI_INVALID_PARAMETER  IoMmuAccess specified an illegal combination of access.
 846   @retval EFI_UNSUPPORTED        The bit mask of IoMmuAccess is not supported by the IOMMU.
 847   @retval EFI_UNSUPPORTED        The IOMMU does not support the memory range specified by BaseAddress and Length.
 848   @retval EFI_OUT_OF_RESOURCES   There are not enough resources available to modify the IOMMU access.
 849   @retval EFI_DEVICE_ERROR       The IOMMU device reported an error while attempting the operation.
 850 **/
 851 EFI_STATUS
 852 SetPageAttribute (
 853   IN UINTN                         VtdIndex,
 854   IN UINT16                        DomainIdentifier,
 855   IN VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry,
 856   IN UINT64                        BaseAddress,
 857   IN UINT64                        Length,
 858   IN UINT64                        IoMmuAccess
 859   )
 860 {
 861   EFI_STATUS Status;
 862   Status = EFI_NOT_FOUND;
 863   if (SecondLevelPagingEntry != NULL) {
 864     Status = SetSecondLevelPagingAttribute (VtdIndex, DomainIdentifier, SecondLevelPagingEntry, BaseAddress, Length, IoMmuAccess);
 865   }
 866   return Status;
 867 }
 868
 869 /**
 870   Set VTd attribute for a system memory.
 871
 872   @param[in]  Segment           The Segment used to identify a VTd engine.
 873   @param[in]  SourceId          The SourceId used to identify a VTd engine and table entry.
 874   @param[in]  BaseAddress       The base of device memory address to be used as the DMA memory.
 875   @param[in]  Length            The length of device memory address to be used as the DMA memory.
 876   @param[in]  IoMmuAccess       The IOMMU access.
 877
 878   @retval EFI_SUCCESS            The IoMmuAccess is set for the memory range specified by BaseAddress and Length.
 879   @retval EFI_INVALID_PARAMETER  BaseAddress is not IoMmu Page size aligned.
 880   @retval EFI_INVALID_PARAMETER  Length is not IoMmu Page size aligned.
 881   @retval EFI_INVALID_PARAMETER  Length is 0.
 882   @retval EFI_INVALID_PARAMETER  IoMmuAccess specified an illegal combination of access.
 883   @retval EFI_UNSUPPORTED        The bit mask of IoMmuAccess is not supported by the IOMMU.
 884   @retval EFI_UNSUPPORTED        The IOMMU does not support the memory range specified by BaseAddress and Length.
 885   @retval EFI_OUT_OF_RESOURCES   There are not enough resources available to modify the IOMMU access.
 886   @retval EFI_DEVICE_ERROR       The IOMMU device reported an error while attempting the operation.
 887 **/
 888 EFI_STATUS
 889 SetAccessAttribute (
 890   IN UINT16                Segment,
 891   IN VTD_SOURCE_ID         SourceId,
 892   IN UINT64                BaseAddress,
 893   IN UINT64                Length,
 894   IN UINT64                IoMmuAccess
 895   )
 896 {
 897   UINTN                         VtdIndex;
 898   EFI_STATUS                    Status;
 899   VTD_EXT_CONTEXT_ENTRY         *ExtContextEntry;
 900   VTD_CONTEXT_ENTRY             *ContextEntry;
 901   VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry;
 902   UINT64                        Pt;
 903   UINTN                         PciDataIndex;
 904   UINT16                        DomainIdentifier;
 905
 906   SecondLevelPagingEntry = NULL;
 907
 908   DEBUG ((DEBUG_VERBOSE,"SetAccessAttribute (S%04x B%02x D%02x F%02x) (0x%016lx - 0x%08x, %x)\n", Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function, BaseAddress, (UINTN)Length, IoMmuAccess));
 909
 910   VtdIndex = FindVtdIndexByPciDevice (Segment, SourceId, &ExtContextEntry, &ContextEntry);
 911   if (VtdIndex == (UINTN)-1) {
 912     DEBUG ((DEBUG_ERROR,"SetAccessAttribute - Pci device (S%04x B%02x D%02x F%02x) not found!\n", Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 913     return EFI_DEVICE_ERROR;
 914   }
 915
 916   PciDataIndex = GetPciDataIndex (VtdIndex, Segment, SourceId);
 917   mVtdUnitInformation[VtdIndex].PciDeviceInfo.PciDeviceData[PciDataIndex].AccessCount++;
 918   //
 919   // DomainId should not be 0.
 920   //
 921   DomainIdentifier = (UINT16)(PciDataIndex + 1);
 922
 923   if (ExtContextEntry != NULL) {
 924     if (ExtContextEntry->Bits.Present == 0) {
 925       SecondLevelPagingEntry = CreateSecondLevelPagingEntry (VtdIndex, 0);
 926       DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry - 0x%x (S%04x B%02x D%02x F%02x) New\n", SecondLevelPagingEntry, Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 927       Pt = (UINT64)RShiftU64 ((UINT64)(UINTN)SecondLevelPagingEntry, 12);
 928
 929       ExtContextEntry->Bits.SecondLevelPageTranslationPointerLo = (UINT32) Pt;
 930       ExtContextEntry->Bits.SecondLevelPageTranslationPointerHi = (UINT32) RShiftU64(Pt, 20);
 931       ExtContextEntry->Bits.DomainIdentifier = DomainIdentifier;
 932       ExtContextEntry->Bits.Present = 1;
 933       FlushPageTableMemory (VtdIndex, (UINTN)ExtContextEntry, sizeof(*ExtContextEntry));
 934       DumpDmarExtContextEntryTable (mVtdUnitInformation[VtdIndex].ExtRootEntryTable);
 935       mVtdUnitInformation[VtdIndex].HasDirtyContext = TRUE;
 936     } else {
 937       SecondLevelPagingEntry = (VOID *)(UINTN)VTD_64BITS_ADDRESS(ExtContextEntry->Bits.SecondLevelPageTranslationPointerLo, ExtContextEntry->Bits.SecondLevelPageTranslationPointerHi);
 938       DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry - 0x%x (S%04x B%02x D%02x F%02x)\n", SecondLevelPagingEntry, Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 939     }
 940   } else if (ContextEntry != NULL) {
 941     if (ContextEntry->Bits.Present == 0) {
 942       SecondLevelPagingEntry = CreateSecondLevelPagingEntry (VtdIndex, 0);
 943       DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry - 0x%x (S%04x B%02x D%02x F%02x) New\n", SecondLevelPagingEntry, Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 944       Pt = (UINT64)RShiftU64 ((UINT64)(UINTN)SecondLevelPagingEntry, 12);
 945
 946       ContextEntry->Bits.SecondLevelPageTranslationPointerLo = (UINT32) Pt;
 947       ContextEntry->Bits.SecondLevelPageTranslationPointerHi = (UINT32) RShiftU64(Pt, 20);
 948       ContextEntry->Bits.DomainIdentifier = DomainIdentifier;
 949       ContextEntry->Bits.Present = 1;
 950       FlushPageTableMemory (VtdIndex, (UINTN)ContextEntry, sizeof(*ContextEntry));
 951       DumpDmarContextEntryTable (mVtdUnitInformation[VtdIndex].RootEntryTable);
 952       mVtdUnitInformation[VtdIndex].HasDirtyContext = TRUE;
 953     } else {
 954       SecondLevelPagingEntry = (VOID *)(UINTN)VTD_64BITS_ADDRESS(ContextEntry->Bits.SecondLevelPageTranslationPointerLo, ContextEntry->Bits.SecondLevelPageTranslationPointerHi);
 955       DEBUG ((DEBUG_VERBOSE,"SecondLevelPagingEntry - 0x%x (S%04x B%02x D%02x F%02x)\n", SecondLevelPagingEntry, Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
 956     }
 957   }
 958
 959   //
 960   // Do not update FixedSecondLevelPagingEntry
 961   //
 962   if (SecondLevelPagingEntry != mVtdUnitInformation[VtdIndex].FixedSecondLevelPagingEntry) {
 963     Status = SetPageAttribute (
 964                VtdIndex,
 965                DomainIdentifier,
 966                SecondLevelPagingEntry,
 967                BaseAddress,
 968                Length,
 969                IoMmuAccess
 970                );
 971     if (EFI_ERROR (Status)) {
 972       DEBUG ((DEBUG_ERROR,"SetPageAttribute - %r\n", Status));
 973       return Status;
 974     }
 975   }
 976
 977   InvalidatePageEntry (VtdIndex);
 978
 979   return EFI_SUCCESS;
 980 }
 981
 982 /**
 983   Always enable the VTd page attribute for the device.
 984
 985   @param[in]  Segment           The Segment used to identify a VTd engine.
 986   @param[in]  SourceId          The SourceId used to identify a VTd engine and table entry.
 987
 988   @retval EFI_SUCCESS           The VTd entry is updated to always enable all DMA access for the specific device.
 989 **/
 990 EFI_STATUS
 991 AlwaysEnablePageAttribute (
 992   IN UINT16                  Segment,
 993   IN VTD_SOURCE_ID           SourceId
 994   )
 995 {
 996   UINTN                         VtdIndex;
 997   VTD_EXT_CONTEXT_ENTRY         *ExtContextEntry;
 998   VTD_CONTEXT_ENTRY             *ContextEntry;
 999   VTD_SECOND_LEVEL_PAGING_ENTRY *SecondLevelPagingEntry;
1000   UINT64                        Pt;
1001
1002   DEBUG ((DEBUG_INFO,"AlwaysEnablePageAttribute (S%04x B%02x D%02x F%02x)\n", Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
1003
1004   VtdIndex = FindVtdIndexByPciDevice (Segment, SourceId, &ExtContextEntry, &ContextEntry);
1005   if (VtdIndex == (UINTN)-1) {
1006     DEBUG ((DEBUG_ERROR,"AlwaysEnablePageAttribute - Pci device (S%04x B%02x D%02x F%02x) not found!\n", Segment, SourceId.Bits.Bus, SourceId.Bits.Device, SourceId.Bits.Function));
1007     return EFI_DEVICE_ERROR;
1008   }
1009
1010   if (mVtdUnitInformation[VtdIndex].FixedSecondLevelPagingEntry == 0) {
1011     DEBUG((DEBUG_INFO, "CreateSecondLevelPagingEntry - %d\n", VtdIndex));
1012     mVtdUnitInformation[VtdIndex].FixedSecondLevelPagingEntry = CreateSecondLevelPagingEntry (VtdIndex, EDKII_IOMMU_ACCESS_READ | EDKII_IOMMU_ACCESS_WRITE);
1013   }
1014
1015   SecondLevelPagingEntry = mVtdUnitInformation[VtdIndex].FixedSecondLevelPagingEntry;
1016   Pt = (UINT64)RShiftU64 ((UINT64)(UINTN)SecondLevelPagingEntry, 12);
1017   if (ExtContextEntry != NULL) {
1018     ExtContextEntry->Bits.SecondLevelPageTranslationPointerLo = (UINT32) Pt;
1019     ExtContextEntry->Bits.SecondLevelPageTranslationPointerHi = (UINT32) RShiftU64(Pt, 20);
1020     ExtContextEntry->Bits.DomainIdentifier = ((1 << (UINT8)((UINTN)mVtdUnitInformation[VtdIndex].CapReg.Bits.ND * 2 + 4)) - 1);
1021     ExtContextEntry->Bits.Present = 1;
1022     FlushPageTableMemory (VtdIndex, (UINTN)ExtContextEntry, sizeof(*ExtContextEntry));
1023   } else if (ContextEntry != NULL) {
1024     ContextEntry->Bits.SecondLevelPageTranslationPointerLo = (UINT32) Pt;
1025     ContextEntry->Bits.SecondLevelPageTranslationPointerHi = (UINT32) RShiftU64(Pt, 20);
1026     ContextEntry->Bits.DomainIdentifier = ((1 << (UINT8)((UINTN)mVtdUnitInformation[VtdIndex].CapReg.Bits.ND * 2 + 4)) - 1);
1027     ContextEntry->Bits.Present = 1;
1028     FlushPageTableMemory (VtdIndex, (UINTN)ContextEntry, sizeof(*ContextEntry));
1029   }
1030
1031   return EFI_SUCCESS;
1032 }