UefiCpuPkg/CpuDxe/CpuDxe.c

   1 /** @file
   2   CPU DXE Module to produce CPU ARCH Protocol.
   3
   4   Copyright (c) 2008 - 2022, Intel Corporation. All rights reserved.<BR>
   5   SPDX-License-Identifier: BSD-2-Clause-Patent
   6
   7 **/
   8
   9 #include "CpuDxe.h"
  10 #include "CpuMp.h"
  11 #include "CpuPageTable.h"
  12
  13 #define CPU_INTERRUPT_NUM  256
  14
  15 //
  16 // Global Variables
  17 //
  18 BOOLEAN     InterruptState = FALSE;
  19 EFI_HANDLE  mCpuHandle     = NULL;
  20 BOOLEAN     mIsFlushingGCD;
  21 BOOLEAN     mIsAllocatingPageTable = FALSE;
  22 UINT64      mValidMtrrAddressMask;
  23 UINT64      mValidMtrrBitsMask;
  24 UINT64      mTimerPeriod = 0;
  25
  26 FIXED_MTRR  mFixedMtrrTable[] = {
  27   {
  28     MSR_IA32_MTRR_FIX64K_00000,
  29     0,
  30     0x10000
  31   },
  32   {
  33     MSR_IA32_MTRR_FIX16K_80000,
  34     0x80000,
  35     0x4000
  36   },
  37   {
  38     MSR_IA32_MTRR_FIX16K_A0000,
  39     0xA0000,
  40     0x4000
  41   },
  42   {
  43     MSR_IA32_MTRR_FIX4K_C0000,
  44     0xC0000,
  45     0x1000
  46   },
  47   {
  48     MSR_IA32_MTRR_FIX4K_C8000,
  49     0xC8000,
  50     0x1000
  51   },
  52   {
  53     MSR_IA32_MTRR_FIX4K_D0000,
  54     0xD0000,
  55     0x1000
  56   },
  57   {
  58     MSR_IA32_MTRR_FIX4K_D8000,
  59     0xD8000,
  60     0x1000
  61   },
  62   {
  63     MSR_IA32_MTRR_FIX4K_E0000,
  64     0xE0000,
  65     0x1000
  66   },
  67   {
  68     MSR_IA32_MTRR_FIX4K_E8000,
  69     0xE8000,
  70     0x1000
  71   },
  72   {
  73     MSR_IA32_MTRR_FIX4K_F0000,
  74     0xF0000,
  75     0x1000
  76   },
  77   {
  78     MSR_IA32_MTRR_FIX4K_F8000,
  79     0xF8000,
  80     0x1000
  81   },
  82 };
  83
  84 EFI_CPU_ARCH_PROTOCOL  gCpu = {
  85   CpuFlushCpuDataCache,
  86   CpuEnableInterrupt,
  87   CpuDisableInterrupt,
  88   CpuGetInterruptState,
  89   CpuInit,
  90   CpuRegisterInterruptHandler,
  91   CpuGetTimerValue,
  92   CpuSetMemoryAttributes,
  93   1,                          // NumberOfTimers
  94   4                           // DmaBufferAlignment
  95 };
  96
  97 //
  98 // CPU Arch Protocol Functions
  99 //
 100
 101 /**
 102   Flush CPU data cache. If the instruction cache is fully coherent
 103   with all DMA operations then function can just return EFI_SUCCESS.
 104
 105   @param  This              Protocol instance structure
 106   @param  Start             Physical address to start flushing from.
 107   @param  Length            Number of bytes to flush. Round up to chipset
 108                             granularity.
 109   @param  FlushType         Specifies the type of flush operation to perform.
 110
 111   @retval EFI_SUCCESS       If cache was flushed
 112   @retval EFI_UNSUPPORTED   If flush type is not supported.
 113   @retval EFI_DEVICE_ERROR  If requested range could not be flushed.
 114
 115 **/
 116 EFI_STATUS
 117 EFIAPI
 118 CpuFlushCpuDataCache (
 119   IN EFI_CPU_ARCH_PROTOCOL  *This,
 120   IN EFI_PHYSICAL_ADDRESS   Start,
 121   IN UINT64                 Length,
 122   IN EFI_CPU_FLUSH_TYPE     FlushType
 123   )
 124 {
 125   if (FlushType == EfiCpuFlushTypeWriteBackInvalidate) {
 126     AsmWbinvd ();
 127     return EFI_SUCCESS;
 128   } else if (FlushType == EfiCpuFlushTypeInvalidate) {
 129     AsmInvd ();
 130     return EFI_SUCCESS;
 131   } else {
 132     return EFI_UNSUPPORTED;
 133   }
 134 }
 135
 136 /**
 137   Enables CPU interrupts.
 138
 139   @param  This              Protocol instance structure
 140
 141   @retval EFI_SUCCESS       If interrupts were enabled in the CPU
 142   @retval EFI_DEVICE_ERROR  If interrupts could not be enabled on the CPU.
 143
 144 **/
 145 EFI_STATUS
 146 EFIAPI
 147 CpuEnableInterrupt (
 148   IN EFI_CPU_ARCH_PROTOCOL  *This
 149   )
 150 {
 151   EnableInterrupts ();
 152
 153   InterruptState = TRUE;
 154   return EFI_SUCCESS;
 155 }
 156
 157 /**
 158   Disables CPU interrupts.
 159
 160   @param  This              Protocol instance structure
 161
 162   @retval EFI_SUCCESS       If interrupts were disabled in the CPU.
 163   @retval EFI_DEVICE_ERROR  If interrupts could not be disabled on the CPU.
 164
 165 **/
 166 EFI_STATUS
 167 EFIAPI
 168 CpuDisableInterrupt (
 169   IN EFI_CPU_ARCH_PROTOCOL  *This
 170   )
 171 {
 172   DisableInterrupts ();
 173
 174   InterruptState = FALSE;
 175   return EFI_SUCCESS;
 176 }
 177
 178 /**
 179   Return the state of interrupts.
 180
 181   @param  This                   Protocol instance structure
 182   @param  State                  Pointer to the CPU's current interrupt state
 183
 184   @retval EFI_SUCCESS            If interrupts were disabled in the CPU.
 185   @retval EFI_INVALID_PARAMETER  State is NULL.
 186
 187 **/
 188 EFI_STATUS
 189 EFIAPI
 190 CpuGetInterruptState (
 191   IN  EFI_CPU_ARCH_PROTOCOL  *This,
 192   OUT BOOLEAN                *State
 193   )
 194 {
 195   if (State == NULL) {
 196     return EFI_INVALID_PARAMETER;
 197   }
 198
 199   *State = InterruptState;
 200   return EFI_SUCCESS;
 201 }
 202
 203 /**
 204   Generates an INIT to the CPU.
 205
 206   @param  This              Protocol instance structure
 207   @param  InitType          Type of CPU INIT to perform
 208
 209   @retval EFI_SUCCESS       If CPU INIT occurred. This value should never be
 210                             seen.
 211   @retval EFI_DEVICE_ERROR  If CPU INIT failed.
 212   @retval EFI_UNSUPPORTED   Requested type of CPU INIT not supported.
 213
 214 **/
 215 EFI_STATUS
 216 EFIAPI
 217 CpuInit (
 218   IN EFI_CPU_ARCH_PROTOCOL  *This,
 219   IN EFI_CPU_INIT_TYPE      InitType
 220   )
 221 {
 222   return EFI_UNSUPPORTED;
 223 }
 224
 225 /**
 226   Registers a function to be called from the CPU interrupt handler.
 227
 228   @param  This                   Protocol instance structure
 229   @param  InterruptType          Defines which interrupt to hook. IA-32
 230                                  valid range is 0x00 through 0xFF
 231   @param  InterruptHandler       A pointer to a function of type
 232                                  EFI_CPU_INTERRUPT_HANDLER that is called
 233                                  when a processor interrupt occurs.  A null
 234                                  pointer is an error condition.
 235
 236   @retval EFI_SUCCESS            If handler installed or uninstalled.
 237   @retval EFI_ALREADY_STARTED    InterruptHandler is not NULL, and a handler
 238                                  for InterruptType was previously installed.
 239   @retval EFI_INVALID_PARAMETER  InterruptHandler is NULL, and a handler for
 240                                  InterruptType was not previously installed.
 241   @retval EFI_UNSUPPORTED        The interrupt specified by InterruptType
 242                                  is not supported.
 243
 244 **/
 245 EFI_STATUS
 246 EFIAPI
 247 CpuRegisterInterruptHandler (
 248   IN EFI_CPU_ARCH_PROTOCOL      *This,
 249   IN EFI_EXCEPTION_TYPE         InterruptType,
 250   IN EFI_CPU_INTERRUPT_HANDLER  InterruptHandler
 251   )
 252 {
 253   return RegisterCpuInterruptHandler (InterruptType, InterruptHandler);
 254 }
 255
 256 /**
 257   Returns a timer value from one of the CPU's internal timers. There is no
 258   inherent time interval between ticks but is a function of the CPU frequency.
 259
 260   @param  This                - Protocol instance structure.
 261   @param  TimerIndex          - Specifies which CPU timer is requested.
 262   @param  TimerValue          - Pointer to the returned timer value.
 263   @param  TimerPeriod         - A pointer to the amount of time that passes
 264                                 in femtoseconds (10-15) for each increment
 265                                 of TimerValue. If TimerValue does not
 266                                 increment at a predictable rate, then 0 is
 267                                 returned.  The amount of time that has
 268                                 passed between two calls to GetTimerValue()
 269                                 can be calculated with the formula
 270                                 (TimerValue2 - TimerValue1) * TimerPeriod.
 271                                 This parameter is optional and may be NULL.
 272
 273   @retval EFI_SUCCESS           - If the CPU timer count was returned.
 274   @retval EFI_UNSUPPORTED       - If the CPU does not have any readable timers.
 275   @retval EFI_DEVICE_ERROR      - If an error occurred while reading the timer.
 276   @retval EFI_INVALID_PARAMETER - TimerIndex is not valid or TimerValue is NULL.
 277
 278 **/
 279 EFI_STATUS
 280 EFIAPI
 281 CpuGetTimerValue (
 282   IN  EFI_CPU_ARCH_PROTOCOL  *This,
 283   IN  UINT32                 TimerIndex,
 284   OUT UINT64                 *TimerValue,
 285   OUT UINT64                 *TimerPeriod OPTIONAL
 286   )
 287 {
 288   UINT64  BeginValue;
 289   UINT64  EndValue;
 290
 291   if (TimerValue == NULL) {
 292     return EFI_INVALID_PARAMETER;
 293   }
 294
 295   if (TimerIndex != 0) {
 296     return EFI_INVALID_PARAMETER;
 297   }
 298
 299   *TimerValue = AsmReadTsc ();
 300
 301   if (TimerPeriod != NULL) {
 302     if (mTimerPeriod == 0) {
 303       //
 304       // Read time stamp counter before and after delay of 100 microseconds
 305       //
 306       BeginValue = AsmReadTsc ();
 307       MicroSecondDelay (100);
 308       EndValue = AsmReadTsc ();
 309       //
 310       // Calculate the actual frequency
 311       //
 312       mTimerPeriod = DivU64x64Remainder (
 313                        MultU64x32 (
 314                          1000 * 1000 * 1000,
 315                          100
 316                          ),
 317                        EndValue - BeginValue,
 318                        NULL
 319                        );
 320     }
 321
 322     *TimerPeriod = mTimerPeriod;
 323   }
 324
 325   return EFI_SUCCESS;
 326 }
 327
 328 /**
 329   A minimal wrapper function that allows MtrrSetAllMtrrs() to be passed to
 330   EFI_MP_SERVICES_PROTOCOL.StartupAllAPs() as Procedure.
 331
 332   @param[in] Buffer  Pointer to an MTRR_SETTINGS object, to be passed to
 333                      MtrrSetAllMtrrs().
 334 **/
 335 VOID
 336 EFIAPI
 337 SetMtrrsFromBuffer (
 338   IN VOID  *Buffer
 339   )
 340 {
 341   MtrrSetAllMtrrs (Buffer);
 342 }
 343
 344 /**
 345   Implementation of SetMemoryAttributes() service of CPU Architecture Protocol.
 346
 347   This function modifies the attributes for the memory region specified by BaseAddress and
 348   Length from their current attributes to the attributes specified by Attributes.
 349
 350   @param  This             The EFI_CPU_ARCH_PROTOCOL instance.
 351   @param  BaseAddress      The physical address that is the start address of a memory region.
 352   @param  Length           The size in bytes of the memory region.
 353   @param  Attributes       The bit mask of attributes to set for the memory region.
 354
 355   @retval EFI_SUCCESS           The attributes were set for the memory region.
 356   @retval EFI_ACCESS_DENIED     The attributes for the memory resource range specified by
 357                                 BaseAddress and Length cannot be modified.
 358   @retval EFI_INVALID_PARAMETER Length is zero.
 359                                 Attributes specified an illegal combination of attributes that
 360                                 cannot be set together.
 361   @retval EFI_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 362                                 the memory resource range.
 363   @retval EFI_UNSUPPORTED       The processor does not support one or more bytes of the memory
 364                                 resource range specified by BaseAddress and Length.
 365                                 The bit mask of attributes is not support for the memory resource
 366                                 range specified by BaseAddress and Length.
 367
 368 **/
 369 EFI_STATUS
 370 EFIAPI
 371 CpuSetMemoryAttributes (
 372   IN EFI_CPU_ARCH_PROTOCOL  *This,
 373   IN EFI_PHYSICAL_ADDRESS   BaseAddress,
 374   IN UINT64                 Length,
 375   IN UINT64                 Attributes
 376   )
 377 {
 378   RETURN_STATUS             Status;
 379   MTRR_MEMORY_CACHE_TYPE    CacheType;
 380   EFI_STATUS                MpStatus;
 381   EFI_MP_SERVICES_PROTOCOL  *MpService;
 382   MTRR_SETTINGS             MtrrSettings;
 383   UINT64                    CacheAttributes;
 384   UINT64                    MemoryAttributes;
 385   MTRR_MEMORY_CACHE_TYPE    CurrentCacheType;
 386
 387   //
 388   // If this function is called because GCD SetMemorySpaceAttributes () is called
 389   // by RefreshGcdMemoryAttributes (), then we are just synchronizing GCD memory
 390   // map with MTRR values. So there is no need to modify MTRRs, just return immediately
 391   // to avoid unnecessary computing.
 392   //
 393   if (mIsFlushingGCD) {
 394     DEBUG ((DEBUG_VERBOSE, "  Flushing GCD\n"));
 395     return EFI_SUCCESS;
 396   }
 397
 398   //
 399   // During memory attributes updating, new pages may be allocated to setup
 400   // smaller granularity of page table. Page allocation action might then cause
 401   // another calling of CpuSetMemoryAttributes() recursively, due to memory
 402   // protection policy configured (such as PcdDxeNxMemoryProtectionPolicy).
 403   // Since this driver will always protect memory used as page table by itself,
 404   // there's no need to apply protection policy requested from memory service.
 405   // So it's safe to just return EFI_SUCCESS if this time of calling is caused
 406   // by page table memory allocation.
 407   //
 408   if (mIsAllocatingPageTable) {
 409     DEBUG ((DEBUG_VERBOSE, "  Allocating page table memory\n"));
 410     return EFI_SUCCESS;
 411   }
 412
 413   CacheAttributes  = Attributes & EFI_CACHE_ATTRIBUTE_MASK;
 414   MemoryAttributes = Attributes & EFI_MEMORY_ATTRIBUTE_MASK;
 415
 416   if (Attributes != (CacheAttributes | MemoryAttributes)) {
 417     return EFI_INVALID_PARAMETER;
 418   }
 419
 420   if (CacheAttributes != 0) {
 421     if (!IsMtrrSupported ()) {
 422       return EFI_UNSUPPORTED;
 423     }
 424
 425     switch (CacheAttributes) {
 426       case EFI_MEMORY_UC:
 427         CacheType = CacheUncacheable;
 428         break;
 429
 430       case EFI_MEMORY_WC:
 431         CacheType = CacheWriteCombining;
 432         break;
 433
 434       case EFI_MEMORY_WT:
 435         CacheType = CacheWriteThrough;
 436         break;
 437
 438       case EFI_MEMORY_WP:
 439         CacheType = CacheWriteProtected;
 440         break;
 441
 442       case EFI_MEMORY_WB:
 443         CacheType = CacheWriteBack;
 444         break;
 445
 446       default:
 447         return EFI_INVALID_PARAMETER;
 448     }
 449
 450     CurrentCacheType = MtrrGetMemoryAttribute (BaseAddress);
 451     if (CurrentCacheType != CacheType) {
 452       //
 453       // call MTRR library function
 454       //
 455       Status = MtrrSetMemoryAttribute (
 456                  BaseAddress,
 457                  Length,
 458                  CacheType
 459                  );
 460
 461       if (!RETURN_ERROR (Status)) {
 462         MpStatus = gBS->LocateProtocol (
 463                           &gEfiMpServiceProtocolGuid,
 464                           NULL,
 465                           (VOID **)&MpService
 466                           );
 467         //
 468         // Synchronize the update with all APs
 469         //
 470         if (!EFI_ERROR (MpStatus)) {
 471           MtrrGetAllMtrrs (&MtrrSettings);
 472           MpStatus = MpService->StartupAllAPs (
 473                                   MpService,          // This
 474                                   SetMtrrsFromBuffer, // Procedure
 475                                   FALSE,              // SingleThread
 476                                   NULL,               // WaitEvent
 477                                   0,                  // TimeoutInMicrosecsond
 478                                   &MtrrSettings,      // ProcedureArgument
 479                                   NULL                // FailedCpuList
 480                                   );
 481           ASSERT (MpStatus == EFI_SUCCESS || MpStatus == EFI_NOT_STARTED);
 482         }
 483       }
 484
 485       if (EFI_ERROR (Status)) {
 486         return Status;
 487       }
 488     }
 489   }
 490
 491   //
 492   // Set memory attribute by page table
 493   //
 494   return AssignMemoryPageAttributes (NULL, BaseAddress, Length, MemoryAttributes, NULL);
 495 }
 496
 497 /**
 498   Initializes the valid bits mask and valid address mask for MTRRs.
 499
 500   This function initializes the valid bits mask and valid address mask for MTRRs.
 501
 502 **/
 503 VOID
 504 InitializeMtrrMask (
 505   VOID
 506   )
 507 {
 508   UINT32  RegEax;
 509   UINT8   PhysicalAddressBits;
 510
 511   AsmCpuid (0x80000000, &RegEax, NULL, NULL, NULL);
 512
 513   if (RegEax >= 0x80000008) {
 514     AsmCpuid (0x80000008, &RegEax, NULL, NULL, NULL);
 515
 516     PhysicalAddressBits = (UINT8)RegEax;
 517   } else {
 518     PhysicalAddressBits = 36;
 519   }
 520
 521   mValidMtrrBitsMask    = LShiftU64 (1, PhysicalAddressBits) - 1;
 522   mValidMtrrAddressMask = mValidMtrrBitsMask & 0xfffffffffffff000ULL;
 523 }
 524
 525 /**
 526   Gets GCD Mem Space type from MTRR Type.
 527
 528   This function gets GCD Mem Space type from MTRR Type.
 529
 530   @param  MtrrAttributes  MTRR memory type
 531
 532   @return GCD Mem Space type
 533
 534 **/
 535 UINT64
 536 GetMemorySpaceAttributeFromMtrrType (
 537   IN UINT8  MtrrAttributes
 538   )
 539 {
 540   switch (MtrrAttributes) {
 541     case MTRR_CACHE_UNCACHEABLE:
 542       return EFI_MEMORY_UC;
 543     case MTRR_CACHE_WRITE_COMBINING:
 544       return EFI_MEMORY_WC;
 545     case MTRR_CACHE_WRITE_THROUGH:
 546       return EFI_MEMORY_WT;
 547     case MTRR_CACHE_WRITE_PROTECTED:
 548       return EFI_MEMORY_WP;
 549     case MTRR_CACHE_WRITE_BACK:
 550       return EFI_MEMORY_WB;
 551     default:
 552       return 0;
 553   }
 554 }
 555
 556 /**
 557   Searches memory descriptors covered by given memory range.
 558
 559   This function searches into the Gcd Memory Space for descriptors
 560   (from StartIndex to EndIndex) that contains the memory range
 561   specified by BaseAddress and Length.
 562
 563   @param  MemorySpaceMap       Gcd Memory Space Map as array.
 564   @param  NumberOfDescriptors  Number of descriptors in map.
 565   @param  BaseAddress          BaseAddress for the requested range.
 566   @param  Length               Length for the requested range.
 567   @param  StartIndex           Start index into the Gcd Memory Space Map.
 568   @param  EndIndex             End index into the Gcd Memory Space Map.
 569
 570   @retval EFI_SUCCESS          Search successfully.
 571   @retval EFI_NOT_FOUND        The requested descriptors does not exist.
 572
 573 **/
 574 EFI_STATUS
 575 SearchGcdMemorySpaces (
 576   IN EFI_GCD_MEMORY_SPACE_DESCRIPTOR  *MemorySpaceMap,
 577   IN UINTN                            NumberOfDescriptors,
 578   IN EFI_PHYSICAL_ADDRESS             BaseAddress,
 579   IN UINT64                           Length,
 580   OUT UINTN                           *StartIndex,
 581   OUT UINTN                           *EndIndex
 582   )
 583 {
 584   UINTN  Index;
 585
 586   *StartIndex = 0;
 587   *EndIndex   = 0;
 588   for (Index = 0; Index < NumberOfDescriptors; Index++) {
 589     if ((BaseAddress >= MemorySpaceMap[Index].BaseAddress) &&
 590         (BaseAddress < MemorySpaceMap[Index].BaseAddress + MemorySpaceMap[Index].Length))
 591     {
 592       *StartIndex = Index;
 593     }
 594
 595     if ((BaseAddress + Length - 1 >= MemorySpaceMap[Index].BaseAddress) &&
 596         (BaseAddress + Length - 1 < MemorySpaceMap[Index].BaseAddress + MemorySpaceMap[Index].Length))
 597     {
 598       *EndIndex = Index;
 599       return EFI_SUCCESS;
 600     }
 601   }
 602
 603   return EFI_NOT_FOUND;
 604 }
 605
 606 /**
 607   Sets the attributes for a specified range in Gcd Memory Space Map.
 608
 609   This function sets the attributes for a specified range in
 610   Gcd Memory Space Map.
 611
 612   @param  MemorySpaceMap       Gcd Memory Space Map as array
 613   @param  NumberOfDescriptors  Number of descriptors in map
 614   @param  BaseAddress          BaseAddress for the range
 615   @param  Length               Length for the range
 616   @param  Attributes           Attributes to set
 617
 618   @retval EFI_SUCCESS          Memory attributes set successfully
 619   @retval EFI_NOT_FOUND        The specified range does not exist in Gcd Memory Space
 620
 621 **/
 622 EFI_STATUS
 623 SetGcdMemorySpaceAttributes (
 624   IN EFI_GCD_MEMORY_SPACE_DESCRIPTOR  *MemorySpaceMap,
 625   IN UINTN                            NumberOfDescriptors,
 626   IN EFI_PHYSICAL_ADDRESS             BaseAddress,
 627   IN UINT64                           Length,
 628   IN UINT64                           Attributes
 629   )
 630 {
 631   EFI_STATUS            Status;
 632   UINTN                 Index;
 633   UINTN                 StartIndex;
 634   UINTN                 EndIndex;
 635   EFI_PHYSICAL_ADDRESS  RegionStart;
 636   UINT64                RegionLength;
 637
 638   //
 639   // Get all memory descriptors covered by the memory range
 640   //
 641   Status = SearchGcdMemorySpaces (
 642              MemorySpaceMap,
 643              NumberOfDescriptors,
 644              BaseAddress,
 645              Length,
 646              &StartIndex,
 647              &EndIndex
 648              );
 649   if (EFI_ERROR (Status)) {
 650     return Status;
 651   }
 652
 653   //
 654   // Go through all related descriptors and set attributes accordingly
 655   //
 656   for (Index = StartIndex; Index <= EndIndex; Index++) {
 657     if (MemorySpaceMap[Index].GcdMemoryType == EfiGcdMemoryTypeNonExistent) {
 658       continue;
 659     }
 660
 661     //
 662     // Calculate the start and end address of the overlapping range
 663     //
 664     if (BaseAddress >= MemorySpaceMap[Index].BaseAddress) {
 665       RegionStart = BaseAddress;
 666     } else {
 667       RegionStart = MemorySpaceMap[Index].BaseAddress;
 668     }
 669
 670     if (BaseAddress + Length - 1 < MemorySpaceMap[Index].BaseAddress + MemorySpaceMap[Index].Length) {
 671       RegionLength = BaseAddress + Length - RegionStart;
 672     } else {
 673       RegionLength = MemorySpaceMap[Index].BaseAddress + MemorySpaceMap[Index].Length - RegionStart;
 674     }
 675
 676     //
 677     // Set memory attributes according to MTRR attribute and the original attribute of descriptor
 678     //
 679     gDS->SetMemorySpaceAttributes (
 680            RegionStart,
 681            RegionLength,
 682            (MemorySpaceMap[Index].Attributes & ~EFI_CACHE_ATTRIBUTE_MASK) | (MemorySpaceMap[Index].Capabilities & Attributes)
 683            );
 684   }
 685
 686   return EFI_SUCCESS;
 687 }
 688
 689 /**
 690   Refreshes the GCD Memory Space attributes according to MTRRs.
 691
 692   This function refreshes the GCD Memory Space attributes according to MTRRs.
 693
 694 **/
 695 VOID
 696 RefreshMemoryAttributesFromMtrr (
 697   VOID
 698   )
 699 {
 700   EFI_STATUS                       Status;
 701   UINTN                            Index;
 702   UINTN                            SubIndex;
 703   UINT64                           RegValue;
 704   EFI_PHYSICAL_ADDRESS             BaseAddress;
 705   UINT64                           Length;
 706   UINT64                           Attributes;
 707   UINT64                           CurrentAttributes;
 708   UINT8                            MtrrType;
 709   UINTN                            NumberOfDescriptors;
 710   EFI_GCD_MEMORY_SPACE_DESCRIPTOR  *MemorySpaceMap;
 711   UINT64                           DefaultAttributes;
 712   VARIABLE_MTRR                    VariableMtrr[MTRR_NUMBER_OF_VARIABLE_MTRR];
 713   MTRR_FIXED_SETTINGS              MtrrFixedSettings;
 714   UINT32                           FirmwareVariableMtrrCount;
 715   UINT8                            DefaultMemoryType;
 716
 717   FirmwareVariableMtrrCount = GetFirmwareVariableMtrrCount ();
 718   ASSERT (FirmwareVariableMtrrCount <= MTRR_NUMBER_OF_VARIABLE_MTRR);
 719
 720   MemorySpaceMap = NULL;
 721
 722   //
 723   // Initialize the valid bits mask and valid address mask for MTRRs
 724   //
 725   InitializeMtrrMask ();
 726
 727   //
 728   // Get the memory attribute of variable MTRRs
 729   //
 730   MtrrGetMemoryAttributeInVariableMtrr (
 731     mValidMtrrBitsMask,
 732     mValidMtrrAddressMask,
 733     VariableMtrr
 734     );
 735
 736   //
 737   // Get the memory space map from GCD
 738   //
 739   Status = gDS->GetMemorySpaceMap (
 740                   &NumberOfDescriptors,
 741                   &MemorySpaceMap
 742                   );
 743   ASSERT_EFI_ERROR (Status);
 744
 745   DefaultMemoryType = (UINT8)MtrrGetDefaultMemoryType ();
 746   DefaultAttributes = GetMemorySpaceAttributeFromMtrrType (DefaultMemoryType);
 747
 748   //
 749   // Set default attributes to all spaces.
 750   //
 751   for (Index = 0; Index < NumberOfDescriptors; Index++) {
 752     if (MemorySpaceMap[Index].GcdMemoryType == EfiGcdMemoryTypeNonExistent) {
 753       continue;
 754     }
 755
 756     gDS->SetMemorySpaceAttributes (
 757            MemorySpaceMap[Index].BaseAddress,
 758            MemorySpaceMap[Index].Length,
 759            (MemorySpaceMap[Index].Attributes & ~EFI_CACHE_ATTRIBUTE_MASK) |
 760            (MemorySpaceMap[Index].Capabilities & DefaultAttributes)
 761            );
 762   }
 763
 764   //
 765   // Go for variable MTRRs with WB attribute
 766   //
 767   for (Index = 0; Index < FirmwareVariableMtrrCount; Index++) {
 768     if (VariableMtrr[Index].Valid &&
 769         (VariableMtrr[Index].Type == MTRR_CACHE_WRITE_BACK))
 770     {
 771       SetGcdMemorySpaceAttributes (
 772         MemorySpaceMap,
 773         NumberOfDescriptors,
 774         VariableMtrr[Index].BaseAddress,
 775         VariableMtrr[Index].Length,
 776         EFI_MEMORY_WB
 777         );
 778     }
 779   }
 780
 781   //
 782   // Go for variable MTRRs with the attribute except for WB and UC attributes
 783   //
 784   for (Index = 0; Index < FirmwareVariableMtrrCount; Index++) {
 785     if (VariableMtrr[Index].Valid &&
 786         (VariableMtrr[Index].Type != MTRR_CACHE_WRITE_BACK) &&
 787         (VariableMtrr[Index].Type != MTRR_CACHE_UNCACHEABLE))
 788     {
 789       Attributes = GetMemorySpaceAttributeFromMtrrType ((UINT8)VariableMtrr[Index].Type);
 790       SetGcdMemorySpaceAttributes (
 791         MemorySpaceMap,
 792         NumberOfDescriptors,
 793         VariableMtrr[Index].BaseAddress,
 794         VariableMtrr[Index].Length,
 795         Attributes
 796         );
 797     }
 798   }
 799
 800   //
 801   // Go for variable MTRRs with UC attribute
 802   //
 803   for (Index = 0; Index < FirmwareVariableMtrrCount; Index++) {
 804     if (VariableMtrr[Index].Valid &&
 805         (VariableMtrr[Index].Type == MTRR_CACHE_UNCACHEABLE))
 806     {
 807       SetGcdMemorySpaceAttributes (
 808         MemorySpaceMap,
 809         NumberOfDescriptors,
 810         VariableMtrr[Index].BaseAddress,
 811         VariableMtrr[Index].Length,
 812         EFI_MEMORY_UC
 813         );
 814     }
 815   }
 816
 817   //
 818   // Go for fixed MTRRs
 819   //
 820   Attributes  = 0;
 821   BaseAddress = 0;
 822   Length      = 0;
 823   MtrrGetFixedMtrr (&MtrrFixedSettings);
 824   for (Index = 0; Index < MTRR_NUMBER_OF_FIXED_MTRR; Index++) {
 825     RegValue = MtrrFixedSettings.Mtrr[Index];
 826     //
 827     // Check for continuous fixed MTRR sections
 828     //
 829     for (SubIndex = 0; SubIndex < 8; SubIndex++) {
 830       MtrrType          = (UINT8)RShiftU64 (RegValue, SubIndex * 8);
 831       CurrentAttributes = GetMemorySpaceAttributeFromMtrrType (MtrrType);
 832       if (Length == 0) {
 833         //
 834         // A new MTRR attribute begins
 835         //
 836         Attributes = CurrentAttributes;
 837       } else {
 838         //
 839         // If fixed MTRR attribute changed, then set memory attribute for previous attribute
 840         //
 841         if (CurrentAttributes != Attributes) {
 842           SetGcdMemorySpaceAttributes (
 843             MemorySpaceMap,
 844             NumberOfDescriptors,
 845             BaseAddress,
 846             Length,
 847             Attributes
 848             );
 849           BaseAddress = mFixedMtrrTable[Index].BaseAddress + mFixedMtrrTable[Index].Length * SubIndex;
 850           Length      = 0;
 851           Attributes  = CurrentAttributes;
 852         }
 853       }
 854
 855       Length += mFixedMtrrTable[Index].Length;
 856     }
 857   }
 858
 859   //
 860   // Handle the last fixed MTRR region
 861   //
 862   SetGcdMemorySpaceAttributes (
 863     MemorySpaceMap,
 864     NumberOfDescriptors,
 865     BaseAddress,
 866     Length,
 867     Attributes
 868     );
 869
 870   //
 871   // Free memory space map allocated by GCD service GetMemorySpaceMap ()
 872   //
 873   if (MemorySpaceMap != NULL) {
 874     FreePool (MemorySpaceMap);
 875   }
 876 }
 877
 878 /**
 879  Check if paging is enabled or not.
 880 **/
 881 BOOLEAN
 882 IsPagingAndPageAddressExtensionsEnabled (
 883   VOID
 884   )
 885 {
 886   IA32_CR0  Cr0;
 887   IA32_CR4  Cr4;
 888
 889   Cr0.UintN = AsmReadCr0 ();
 890   Cr4.UintN = AsmReadCr4 ();
 891
 892   return ((Cr0.Bits.PG != 0) && (Cr4.Bits.PAE != 0));
 893 }
 894
 895 /**
 896   Refreshes the GCD Memory Space attributes according to MTRRs and Paging.
 897
 898   This function refreshes the GCD Memory Space attributes according to MTRRs
 899   and page tables.
 900
 901 **/
 902 VOID
 903 RefreshGcdMemoryAttributes (
 904   VOID
 905   )
 906 {
 907   mIsFlushingGCD = TRUE;
 908
 909   if (IsMtrrSupported ()) {
 910     RefreshMemoryAttributesFromMtrr ();
 911   }
 912
 913   if (IsPagingAndPageAddressExtensionsEnabled ()) {
 914     RefreshGcdMemoryAttributesFromPaging ();
 915   }
 916
 917   mIsFlushingGCD = FALSE;
 918 }
 919
 920 /**
 921   Initialize Interrupt Descriptor Table for interrupt handling.
 922
 923 **/
 924 VOID
 925 InitInterruptDescriptorTable (
 926   VOID
 927   )
 928 {
 929   EFI_STATUS                Status;
 930   EFI_VECTOR_HANDOFF_INFO   *VectorInfoList;
 931   EFI_VECTOR_HANDOFF_INFO   *VectorInfo;
 932   IA32_IDT_GATE_DESCRIPTOR  *IdtTable;
 933   IA32_DESCRIPTOR           IdtDescriptor;
 934   UINTN                     IdtEntryCount;
 935
 936   VectorInfo = NULL;
 937   Status     = EfiGetSystemConfigurationTable (&gEfiVectorHandoffTableGuid, (VOID **)&VectorInfoList);
 938   if ((Status == EFI_SUCCESS) && (VectorInfoList != NULL)) {
 939     VectorInfo = VectorInfoList;
 940   }
 941
 942   AsmReadIdtr (&IdtDescriptor);
 943   IdtEntryCount = (IdtDescriptor.Limit + 1) / sizeof (IA32_IDT_GATE_DESCRIPTOR);
 944   if (IdtEntryCount < CPU_INTERRUPT_NUM) {
 945     //
 946     // Increase Interrupt Descriptor Table and Copy the old IDT table in
 947     //
 948     IdtTable = AllocateZeroPool (sizeof (IA32_IDT_GATE_DESCRIPTOR) * CPU_INTERRUPT_NUM);
 949     ASSERT (IdtTable != NULL);
 950     CopyMem (IdtTable, (VOID *)IdtDescriptor.Base, sizeof (IA32_IDT_GATE_DESCRIPTOR) * IdtEntryCount);
 951
 952     //
 953     // Load Interrupt Descriptor Table
 954     //
 955     IdtDescriptor.Base  = (UINTN)IdtTable;
 956     IdtDescriptor.Limit = (UINT16)(sizeof (IA32_IDT_GATE_DESCRIPTOR) * CPU_INTERRUPT_NUM - 1);
 957     AsmWriteIdtr (&IdtDescriptor);
 958   }
 959
 960   Status = InitializeCpuExceptionHandlers (VectorInfo);
 961   ASSERT_EFI_ERROR (Status);
 962 }
 963
 964 /**
 965   Callback function for idle events.
 966
 967   @param  Event                 Event whose notification function is being invoked.
 968   @param  Context               The pointer to the notification function's context,
 969                                 which is implementation-dependent.
 970
 971 **/
 972 VOID
 973 EFIAPI
 974 IdleLoopEventCallback (
 975   IN EFI_EVENT  Event,
 976   IN VOID       *Context
 977   )
 978 {
 979   CpuSleep ();
 980 }
 981
 982 /**
 983   Ensure the compatibility of a memory space descriptor with the MMIO aperture.
 984
 985   The memory space descriptor can come from the GCD memory space map, or it can
 986   represent a gap between two neighboring memory space descriptors. In the
 987   latter case, the GcdMemoryType field is expected to be
 988   EfiGcdMemoryTypeNonExistent.
 989
 990   If the memory space descriptor already has type
 991   EfiGcdMemoryTypeMemoryMappedIo, and its capabilities are a superset of the
 992   required capabilities, then no action is taken -- it is by definition
 993   compatible with the aperture.
 994
 995   Otherwise, the intersection of the memory space descriptor is calculated with
 996   the aperture. If the intersection is the empty set (no overlap), no action is
 997   taken; the memory space descriptor is compatible with the aperture.
 998
 999   Otherwise, the type of the descriptor is investigated again. If the type is
1000   EfiGcdMemoryTypeNonExistent (representing a gap, or a genuine descriptor with
1001   such a type), then an attempt is made to add the intersection as MMIO space
1002   to the GCD memory space map, with the specified capabilities. This ensures
1003   continuity for the aperture, and the descriptor is deemed compatible with the
1004   aperture.
1005
1006   Otherwise, the memory space descriptor is incompatible with the MMIO
1007   aperture.
1008
1009   @param[in] Base         Base address of the aperture.
1010   @param[in] Length       Length of the aperture.
1011   @param[in] Capabilities Capabilities required by the aperture.
1012   @param[in] Descriptor   The descriptor to ensure compatibility with the
1013                           aperture for.
1014
1015   @retval EFI_SUCCESS            The descriptor is compatible. The GCD memory
1016                                  space map may have been updated, for
1017                                  continuity within the aperture.
1018   @retval EFI_INVALID_PARAMETER  The descriptor is incompatible.
1019   @return                        Error codes from gDS->AddMemorySpace().
1020 **/
1021 EFI_STATUS
1022 IntersectMemoryDescriptor (
1023   IN  UINT64                                 Base,
1024   IN  UINT64                                 Length,
1025   IN  UINT64                                 Capabilities,
1026   IN  CONST EFI_GCD_MEMORY_SPACE_DESCRIPTOR  *Descriptor
1027   )
1028 {
1029   UINT64      IntersectionBase;
1030   UINT64      IntersectionEnd;
1031   EFI_STATUS  Status;
1032
1033   if ((Descriptor->GcdMemoryType == EfiGcdMemoryTypeMemoryMappedIo) &&
1034       ((Descriptor->Capabilities & Capabilities) == Capabilities))
1035   {
1036     return EFI_SUCCESS;
1037   }
1038
1039   IntersectionBase = MAX (Base, Descriptor->BaseAddress);
1040   IntersectionEnd  = MIN (
1041                        Base + Length,
1042                        Descriptor->BaseAddress + Descriptor->Length
1043                        );
1044   if (IntersectionBase >= IntersectionEnd) {
1045     //
1046     // The descriptor and the aperture don't overlap.
1047     //
1048     return EFI_SUCCESS;
1049   }
1050
1051   if (Descriptor->GcdMemoryType == EfiGcdMemoryTypeNonExistent) {
1052     Status = gDS->AddMemorySpace (
1053                     EfiGcdMemoryTypeMemoryMappedIo,
1054                     IntersectionBase,
1055                     IntersectionEnd - IntersectionBase,
1056                     Capabilities
1057                     );
1058
1059     DEBUG ((
1060       EFI_ERROR (Status) ? DEBUG_ERROR : DEBUG_VERBOSE,
1061       "%a: %a: add [%Lx, %Lx): %r\n",
1062       gEfiCallerBaseName,
1063       __FUNCTION__,
1064       IntersectionBase,
1065       IntersectionEnd,
1066       Status
1067       ));
1068     return Status;
1069   }
1070
1071   DEBUG ((
1072     DEBUG_ERROR,
1073     "%a: %a: desc [%Lx, %Lx) type %u cap %Lx conflicts "
1074     "with aperture [%Lx, %Lx) cap %Lx\n",
1075     gEfiCallerBaseName,
1076     __FUNCTION__,
1077     Descriptor->BaseAddress,
1078     Descriptor->BaseAddress + Descriptor->Length,
1079     (UINT32)Descriptor->GcdMemoryType,
1080     Descriptor->Capabilities,
1081     Base,
1082     Base + Length,
1083     Capabilities
1084     ));
1085   return EFI_INVALID_PARAMETER;
1086 }
1087
1088 /**
1089   Add MMIO space to GCD.
1090   The routine checks the GCD database and only adds those which are
1091   not added in the specified range to GCD.
1092
1093   @param Base         Base address of the MMIO space.
1094   @param Length       Length of the MMIO space.
1095   @param Capabilities Capabilities of the MMIO space.
1096
1097   @retval EFI_SUCCESS The MMIO space was added successfully.
1098 **/
1099 EFI_STATUS
1100 AddMemoryMappedIoSpace (
1101   IN  UINT64  Base,
1102   IN  UINT64  Length,
1103   IN  UINT64  Capabilities
1104   )
1105 {
1106   EFI_STATUS                       Status;
1107   UINTN                            Index;
1108   UINTN                            NumberOfDescriptors;
1109   EFI_GCD_MEMORY_SPACE_DESCRIPTOR  *MemorySpaceMap;
1110
1111   Status = gDS->GetMemorySpaceMap (&NumberOfDescriptors, &MemorySpaceMap);
1112   if (EFI_ERROR (Status)) {
1113     DEBUG ((
1114       DEBUG_ERROR,
1115       "%a: %a: GetMemorySpaceMap(): %r\n",
1116       gEfiCallerBaseName,
1117       __FUNCTION__,
1118       Status
1119       ));
1120     return Status;
1121   }
1122
1123   for (Index = 0; Index < NumberOfDescriptors; Index++) {
1124     Status = IntersectMemoryDescriptor (
1125                Base,
1126                Length,
1127                Capabilities,
1128                &MemorySpaceMap[Index]
1129                );
1130     if (EFI_ERROR (Status)) {
1131       goto FreeMemorySpaceMap;
1132     }
1133   }
1134
1135   DEBUG_CODE_BEGIN ();
1136   //
1137   // Make sure there are adjacent descriptors covering [Base, Base + Length).
1138   // It is possible that they have not been merged; merging can be prevented
1139   // by allocation and different capabilities.
1140   //
1141   UINT64                           CheckBase;
1142   EFI_STATUS                       CheckStatus;
1143   EFI_GCD_MEMORY_SPACE_DESCRIPTOR  Descriptor;
1144
1145   for (CheckBase = Base;
1146        CheckBase < Base + Length;
1147        CheckBase = Descriptor.BaseAddress + Descriptor.Length)
1148   {
1149     CheckStatus = gDS->GetMemorySpaceDescriptor (CheckBase, &Descriptor);
1150     ASSERT_EFI_ERROR (CheckStatus);
1151     ASSERT (Descriptor.GcdMemoryType == EfiGcdMemoryTypeMemoryMappedIo);
1152     ASSERT ((Descriptor.Capabilities & Capabilities) == Capabilities);
1153   }
1154
1155   DEBUG_CODE_END ();
1156
1157 FreeMemorySpaceMap:
1158   FreePool (MemorySpaceMap);
1159
1160   return Status;
1161 }
1162
1163 /**
1164   Add and allocate CPU local APIC memory mapped space.
1165
1166   @param[in]ImageHandle     Image handle this driver.
1167
1168 **/
1169 VOID
1170 AddLocalApicMemorySpace (
1171   IN EFI_HANDLE  ImageHandle
1172   )
1173 {
1174   EFI_STATUS            Status;
1175   EFI_PHYSICAL_ADDRESS  BaseAddress;
1176
1177   BaseAddress = (EFI_PHYSICAL_ADDRESS)GetLocalApicBaseAddress ();
1178   Status      = AddMemoryMappedIoSpace (BaseAddress, SIZE_4KB, EFI_MEMORY_UC);
1179   ASSERT_EFI_ERROR (Status);
1180
1181   //
1182   // Try to allocate APIC memory mapped space, does not check return
1183   // status because it may be allocated by other driver, or DXE Core if
1184   // this range is built into Memory Allocation HOB.
1185   //
1186   Status = gDS->AllocateMemorySpace (
1187                   EfiGcdAllocateAddress,
1188                   EfiGcdMemoryTypeMemoryMappedIo,
1189                   0,
1190                   SIZE_4KB,
1191                   &BaseAddress,
1192                   ImageHandle,
1193                   NULL
1194                   );
1195   if (EFI_ERROR (Status)) {
1196     DEBUG ((
1197       DEBUG_INFO,
1198       "%a: %a: AllocateMemorySpace() Status - %r\n",
1199       gEfiCallerBaseName,
1200       __FUNCTION__,
1201       Status
1202       ));
1203   }
1204 }
1205
1206 /**
1207   Initialize the state information for the CPU Architectural Protocol.
1208
1209   @param ImageHandle     Image handle this driver.
1210   @param SystemTable     Pointer to the System Table.
1211
1212   @retval EFI_SUCCESS           Thread can be successfully created
1213   @retval EFI_OUT_OF_RESOURCES  Cannot allocate protocol data structure
1214   @retval EFI_DEVICE_ERROR      Cannot create the thread
1215
1216 **/
1217 EFI_STATUS
1218 EFIAPI
1219 InitializeCpu (
1220   IN EFI_HANDLE        ImageHandle,
1221   IN EFI_SYSTEM_TABLE  *SystemTable
1222   )
1223 {
1224   EFI_STATUS  Status;
1225   EFI_EVENT   IdleLoopEvent;
1226
1227   InitializePageTableLib ();
1228
1229   InitializeFloatingPointUnits ();
1230
1231   //
1232   // Make sure interrupts are disabled
1233   //
1234   DisableInterrupts ();
1235
1236   //
1237   // Init GDT for DXE
1238   //
1239   InitGlobalDescriptorTable ();
1240
1241   //
1242   // Setup IDT pointer, IDT and interrupt entry points
1243   //
1244   InitInterruptDescriptorTable ();
1245
1246   //
1247   // Install CPU Architectural Protocol
1248   //
1249   Status = gBS->InstallMultipleProtocolInterfaces (
1250                   &mCpuHandle,
1251                   &gEfiCpuArchProtocolGuid,
1252                   &gCpu,
1253                   NULL
1254                   );
1255   ASSERT_EFI_ERROR (Status);
1256
1257   //
1258   // Refresh GCD memory space map according to MTRR value.
1259   //
1260   RefreshGcdMemoryAttributes ();
1261
1262   //
1263   // Add and allocate local APIC memory mapped space
1264   //
1265   AddLocalApicMemorySpace (ImageHandle);
1266
1267   //
1268   // Setup a callback for idle events
1269   //
1270   Status = gBS->CreateEventEx (
1271                   EVT_NOTIFY_SIGNAL,
1272                   TPL_NOTIFY,
1273                   IdleLoopEventCallback,
1274                   NULL,
1275                   &gIdleLoopEventGuid,
1276                   &IdleLoopEvent
1277                   );
1278   ASSERT_EFI_ERROR (Status);
1279
1280   InitializeMpSupport ();
1281
1282   return Status;
1283 }