[mirror_edk2.git] / UefiCpuPkg / PiSmmCpuDxeSmm / MpService.c

/** @file\r
SMM MP service implementation\r
\r
Copyright (c) 2009 - 2019, Intel Corporation. All rights reserved.<BR>\r
Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>\r
\r
SPDX-License-Identifier: BSD-2-Clause-Patent\r
\r
**/\r
\r
#include "PiSmmCpuDxeSmm.h"\r
\r
//\r
// Slots for all MTRR( FIXED MTRR + VARIABLE MTRR + MTRR_LIB_IA32_MTRR_DEF_TYPE)\r
//\r
MTRR_SETTINGS                               gSmiMtrrs;\r
UINT64                                      gPhyMask;\r
SMM_DISPATCHER_MP_SYNC_DATA                 *mSmmMpSyncData = NULL;\r
UINTN                                       mSmmMpSyncDataSize;\r
SMM_CPU_SEMAPHORES                          mSmmCpuSemaphores;\r
UINTN                                       mSemaphoreSize;\r
SPIN_LOCK                                   *mPFLock = NULL;\r
SMM_CPU_SYNC_MODE                           mCpuSmmSyncMode;\r
BOOLEAN                                     mMachineCheckSupported = FALSE;\r
\r
/**\r
  Performs an atomic compare exchange operation to get semaphore.\r
  The compare exchange operation must be performed using\r
  MP safe mechanisms.\r
\r
  @param      Sem        IN:  32-bit unsigned integer\r
                         OUT: original integer - 1\r
  @return     Original integer - 1\r
\r
**/\r
UINT32\r
WaitForSemaphore (\r
  IN OUT  volatile UINT32           *Sem\r
  )\r
{\r
  UINT32                            Value;\r
\r
  do {\r
    Value = *Sem;\r
  } while (Value == 0 ||\r
           InterlockedCompareExchange32 (\r
             (UINT32*)Sem,\r
             Value,\r
             Value - 1\r
             ) != Value);\r
  return Value - 1;\r
}\r
\r
\r
/**\r
  Performs an atomic compare exchange operation to release semaphore.\r
  The compare exchange operation must be performed using\r
  MP safe mechanisms.\r
\r
  @param      Sem        IN:  32-bit unsigned integer\r
                         OUT: original integer + 1\r
  @return     Original integer + 1\r
\r
**/\r
UINT32\r
ReleaseSemaphore (\r
  IN OUT  volatile UINT32           *Sem\r
  )\r
{\r
  UINT32                            Value;\r
\r
  do {\r
    Value = *Sem;\r
  } while (Value + 1 != 0 &&\r
           InterlockedCompareExchange32 (\r
             (UINT32*)Sem,\r
             Value,\r
             Value + 1\r
             ) != Value);\r
  return Value + 1;\r
}\r
\r
/**\r
  Performs an atomic compare exchange operation to lock semaphore.\r
  The compare exchange operation must be performed using\r
  MP safe mechanisms.\r
\r
  @param      Sem        IN:  32-bit unsigned integer\r
                         OUT: -1\r
  @return     Original integer\r
\r
**/\r
UINT32\r
LockdownSemaphore (\r
  IN OUT  volatile UINT32           *Sem\r
  )\r
{\r
  UINT32                            Value;\r
\r
  do {\r
    Value = *Sem;\r
  } while (InterlockedCompareExchange32 (\r
             (UINT32*)Sem,\r
             Value, (UINT32)-1\r
             ) != Value);\r
  return Value;\r
}\r
\r
/**\r
  Wait all APs to performs an atomic compare exchange operation to release semaphore.\r
\r
  @param   NumberOfAPs      AP number\r
\r
**/\r
VOID\r
WaitForAllAPs (\r
  IN      UINTN                     NumberOfAPs\r
  )\r
{\r
  UINTN                             BspIndex;\r
\r
  BspIndex = mSmmMpSyncData->BspIndex;\r
  while (NumberOfAPs-- > 0) {\r
    WaitForSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);\r
  }\r
}\r
\r
/**\r
  Performs an atomic compare exchange operation to release semaphore\r
  for each AP.\r
\r
**/\r
VOID\r
ReleaseAllAPs (\r
  VOID\r
  )\r
{\r
  UINTN                             Index;\r
  UINTN                             BspIndex;\r
\r
  BspIndex = mSmmMpSyncData->BspIndex;\r
  for (Index = mMaxNumberOfCpus; Index-- > 0;) {\r
    if (Index != BspIndex && *(mSmmMpSyncData->CpuData[Index].Present)) {\r
      ReleaseSemaphore (mSmmMpSyncData->CpuData[Index].Run);\r
    }\r
  }\r
}\r
\r
/**\r
  Checks if all CPUs (with certain exceptions) have checked in for this SMI run\r
\r
  @param   Exceptions     CPU Arrival exception flags.\r
\r
  @retval   TRUE  if all CPUs the have checked in.\r
  @retval   FALSE  if at least one Normal AP hasn't checked in.\r
\r
**/\r
BOOLEAN\r
AllCpusInSmmWithExceptions (\r
  SMM_CPU_ARRIVAL_EXCEPTIONS  Exceptions\r
  )\r
{\r
  UINTN                             Index;\r
  SMM_CPU_DATA_BLOCK                *CpuData;\r
  EFI_PROCESSOR_INFORMATION         *ProcessorInfo;\r
\r
  ASSERT (*mSmmMpSyncData->Counter <= mNumberOfCpus);\r
\r
  if (*mSmmMpSyncData->Counter == mNumberOfCpus) {\r
    return TRUE;\r
  }\r
\r
  CpuData = mSmmMpSyncData->CpuData;\r
  ProcessorInfo = gSmmCpuPrivate->ProcessorInfo;\r
  for (Index = mMaxNumberOfCpus; Index-- > 0;) {\r
    if (!(*(CpuData[Index].Present)) && ProcessorInfo[Index].ProcessorId != INVALID_APIC_ID) {\r
      if (((Exceptions & ARRIVAL_EXCEPTION_DELAYED) != 0) && SmmCpuFeaturesGetSmmRegister (Index, SmmRegSmmDelayed) != 0) {\r
        continue;\r
      }\r
      if (((Exceptions & ARRIVAL_EXCEPTION_BLOCKED) != 0) && SmmCpuFeaturesGetSmmRegister (Index, SmmRegSmmBlocked) != 0) {\r
        continue;\r
      }\r
      if (((Exceptions & ARRIVAL_EXCEPTION_SMI_DISABLED) != 0) && SmmCpuFeaturesGetSmmRegister (Index, SmmRegSmmEnable) != 0) {\r
        continue;\r
      }\r
      return FALSE;\r
    }\r
  }\r
\r
\r
  return TRUE;\r
}\r
\r
/**\r
  Has OS enabled Lmce in the MSR_IA32_MCG_EXT_CTL\r
\r
  @retval TRUE     Os enable lmce.\r
  @retval FALSE    Os not enable lmce.\r
\r
**/\r
BOOLEAN\r
IsLmceOsEnabled (\r
  VOID\r
  )\r
{\r
  MSR_IA32_MCG_CAP_REGISTER          McgCap;\r
  MSR_IA32_FEATURE_CONTROL_REGISTER  FeatureCtrl;\r
  MSR_IA32_MCG_EXT_CTL_REGISTER      McgExtCtrl;\r
\r
  McgCap.Uint64 = AsmReadMsr64 (MSR_IA32_MCG_CAP);\r
  if (McgCap.Bits.MCG_LMCE_P == 0) {\r
    return FALSE;\r
  }\r
\r
  FeatureCtrl.Uint64 = AsmReadMsr64 (MSR_IA32_FEATURE_CONTROL);\r
  if (FeatureCtrl.Bits.LmceOn == 0) {\r
    return FALSE;\r
  }\r
\r
  McgExtCtrl.Uint64 = AsmReadMsr64 (MSR_IA32_MCG_EXT_CTL);\r
  return (BOOLEAN) (McgExtCtrl.Bits.LMCE_EN == 1);\r
}\r
\r
/**\r
  Return if Local machine check exception signaled.\r
\r
  Indicates (when set) that a local machine check exception was generated. This indicates that the current machine-check event was\r
  delivered to only the logical processor.\r
\r
  @retval TRUE    LMCE was signaled.\r
  @retval FALSE   LMCE was not signaled.\r
\r
**/\r
BOOLEAN\r
IsLmceSignaled (\r
  VOID\r
  )\r
{\r
  MSR_IA32_MCG_STATUS_REGISTER McgStatus;\r
\r
  McgStatus.Uint64 = AsmReadMsr64 (MSR_IA32_MCG_STATUS);\r
  return (BOOLEAN) (McgStatus.Bits.LMCE_S == 1);\r
}\r
\r
/**\r
  Given timeout constraint, wait for all APs to arrive, and insure when this function returns, no AP will execute normal mode code before\r
  entering SMM, except SMI disabled APs.\r
\r
**/\r
VOID\r
SmmWaitForApArrival (\r
  VOID\r
  )\r
{\r
  UINT64                            Timer;\r
  UINTN                             Index;\r
  BOOLEAN                           LmceEn;\r
  BOOLEAN                           LmceSignal;\r
\r
  ASSERT (*mSmmMpSyncData->Counter <= mNumberOfCpus);\r
\r
  LmceEn     = FALSE;\r
  LmceSignal = FALSE;\r
  if (mMachineCheckSupported) {\r
    LmceEn     = IsLmceOsEnabled ();\r
    LmceSignal = IsLmceSignaled();\r
  }\r
\r
  //\r
  // Platform implementor should choose a timeout value appropriately:\r
  // - The timeout value should balance the SMM time constrains and the likelihood that delayed CPUs are excluded in the SMM run. Note\r
  //   the SMI Handlers must ALWAYS take into account the cases that not all APs are available in an SMI run.\r
  // - The timeout value must, in the case of 2nd timeout, be at least long enough to give time for all APs to receive the SMI IPI\r
  //   and either enter SMM or buffer the SMI, to insure there is no CPU running normal mode code when SMI handling starts. This will\r
  //   be TRUE even if a blocked CPU is brought out of the blocked state by a normal mode CPU (before the normal mode CPU received the\r
  //   SMI IPI), because with a buffered SMI, and CPU will enter SMM immediately after it is brought out of the blocked state.\r
  // - The timeout value must be longer than longest possible IO operation in the system\r
  //\r
\r
  //\r
  // Sync with APs 1st timeout\r
  //\r
  for (Timer = StartSyncTimer ();\r
       !IsSyncTimerTimeout (Timer) && !(LmceEn && LmceSignal) &&\r
       !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED | ARRIVAL_EXCEPTION_SMI_DISABLED );\r
       ) {\r
    CpuPause ();\r
  }\r
\r
  //\r
  // Not all APs have arrived, so we need 2nd round of timeout. IPIs should be sent to ALL none present APs,\r
  // because:\r
  // a) Delayed AP may have just come out of the delayed state. Blocked AP may have just been brought out of blocked state by some AP running\r
  //    normal mode code. These APs need to be guaranteed to have an SMI pending to insure that once they are out of delayed / blocked state, they\r
  //    enter SMI immediately without executing instructions in normal mode. Note traditional flow requires there are no APs doing normal mode\r
  //    work while SMI handling is on-going.\r
  // b) As a consequence of SMI IPI sending, (spurious) SMI may occur after this SMM run.\r
  // c) ** NOTE **: Use SMI disabling feature VERY CAREFULLY (if at all) for traditional flow, because a processor in SMI-disabled state\r
  //    will execute normal mode code, which breaks the traditional SMI handlers' assumption that no APs are doing normal\r
  //    mode work while SMI handling is on-going.\r
  // d) We don't add code to check SMI disabling status to skip sending IPI to SMI disabled APs, because:\r
  //    - In traditional flow, SMI disabling is discouraged.\r
  //    - In relaxed flow, CheckApArrival() will check SMI disabling status before calling this function.\r
  //    In both cases, adding SMI-disabling checking code increases overhead.\r
  //\r
  if (*mSmmMpSyncData->Counter < mNumberOfCpus) {\r
    //\r
    // Send SMI IPIs to bring outside processors in\r
    //\r
    for (Index = mMaxNumberOfCpus; Index-- > 0;) {\r
      if (!(*(mSmmMpSyncData->CpuData[Index].Present)) && gSmmCpuPrivate->ProcessorInfo[Index].ProcessorId != INVALID_APIC_ID) {\r
        SendSmiIpi ((UINT32)gSmmCpuPrivate->ProcessorInfo[Index].ProcessorId);\r
      }\r
    }\r
\r
    //\r
    // Sync with APs 2nd timeout.\r
    //\r
    for (Timer = StartSyncTimer ();\r
         !IsSyncTimerTimeout (Timer) &&\r
         !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED | ARRIVAL_EXCEPTION_SMI_DISABLED );\r
         ) {\r
      CpuPause ();\r
    }\r
  }\r
\r
  return;\r
}\r
\r
\r
/**\r
  Replace OS MTRR's with SMI MTRR's.\r
\r
  @param    CpuIndex             Processor Index\r
\r
**/\r
VOID\r
ReplaceOSMtrrs (\r
  IN      UINTN                     CpuIndex\r
  )\r
{\r
  SmmCpuFeaturesDisableSmrr ();\r
\r
  //\r
  // Replace all MTRRs registers\r
  //\r
  MtrrSetAllMtrrs (&gSmiMtrrs);\r
}\r
\r
/**\r
  SMI handler for BSP.\r
\r
  @param     CpuIndex         BSP processor Index\r
  @param     SyncMode         SMM MP sync mode\r
\r
**/\r
VOID\r
BSPHandler (\r
  IN      UINTN                     CpuIndex,\r
  IN      SMM_CPU_SYNC_MODE         SyncMode\r
  )\r
{\r
  UINTN                             Index;\r
  MTRR_SETTINGS                     Mtrrs;\r
  UINTN                             ApCount;\r
  BOOLEAN                           ClearTopLevelSmiResult;\r
  UINTN                             PresentCount;\r
\r
  ASSERT (CpuIndex == mSmmMpSyncData->BspIndex);\r
  ApCount = 0;\r
\r
  //\r
  // Flag BSP's presence\r
  //\r
  *mSmmMpSyncData->InsideSmm = TRUE;\r
\r
  //\r
  // Initialize Debug Agent to start source level debug in BSP handler\r
  //\r
  InitializeDebugAgent (DEBUG_AGENT_INIT_ENTER_SMI, NULL, NULL);\r
\r
  //\r
  // Mark this processor's presence\r
  //\r
  *(mSmmMpSyncData->CpuData[CpuIndex].Present) = TRUE;\r
\r
  //\r
  // Clear platform top level SMI status bit before calling SMI handlers. If\r
  // we cleared it after SMI handlers are run, we would miss the SMI that\r
  // occurs after SMI handlers are done and before SMI status bit is cleared.\r
  //\r
  ClearTopLevelSmiResult = ClearTopLevelSmiStatus();\r
  ASSERT (ClearTopLevelSmiResult == TRUE);\r
\r
  //\r
  // Set running processor index\r
  //\r
  gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu = CpuIndex;\r
\r
  //\r
  // If Traditional Sync Mode or need to configure MTRRs: gather all available APs.\r
  //\r
  if (SyncMode == SmmCpuSyncModeTradition || SmmCpuFeaturesNeedConfigureMtrrs()) {\r
\r
    //\r
    // Wait for APs to arrive\r
    //\r
    SmmWaitForApArrival();\r
\r
    //\r
    // Lock the counter down and retrieve the number of APs\r
    //\r
    *mSmmMpSyncData->AllCpusInSync = TRUE;\r
    ApCount = LockdownSemaphore (mSmmMpSyncData->Counter) - 1;\r
\r
    //\r
    // Wait for all APs to get ready for programming MTRRs\r
    //\r
    WaitForAllAPs (ApCount);\r
\r
    if (SmmCpuFeaturesNeedConfigureMtrrs()) {\r
      //\r
      // Signal all APs it's time for backup MTRRs\r
      //\r
      ReleaseAllAPs ();\r
\r
      //\r
      // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at\r
      // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set\r
      // to a large enough value to avoid this situation.\r
      // Note: For HT capable CPUs, threads within a core share the same set of MTRRs.\r
      // We do the backup first and then set MTRR to avoid race condition for threads\r
      // in the same core.\r
      //\r
      MtrrGetAllMtrrs(&Mtrrs);\r
\r
      //\r
      // Wait for all APs to complete their MTRR saving\r
      //\r
      WaitForAllAPs (ApCount);\r
\r
      //\r
      // Let all processors program SMM MTRRs together\r
      //\r
      ReleaseAllAPs ();\r
\r
      //\r
      // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at\r
      // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set\r
      // to a large enough value to avoid this situation.\r
      //\r
      ReplaceOSMtrrs (CpuIndex);\r
\r
      //\r
      // Wait for all APs to complete their MTRR programming\r
      //\r
      WaitForAllAPs (ApCount);\r
    }\r
  }\r
\r
  //\r
  // The BUSY lock is initialized to Acquired state\r
  //\r
  AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);\r
\r
  //\r
  // Perform the pre tasks\r
  //\r
  PerformPreTasks ();\r
\r
  //\r
  // Invoke SMM Foundation EntryPoint with the processor information context.\r
  //\r
  gSmmCpuPrivate->SmmCoreEntry (&gSmmCpuPrivate->SmmCoreEntryContext);\r
\r
  //\r
  // Make sure all APs have completed their pending none-block tasks\r
  //\r
  for (Index = mMaxNumberOfCpus; Index-- > 0;) {\r
    if (Index != CpuIndex && *(mSmmMpSyncData->CpuData[Index].Present)) {\r
      AcquireSpinLock (mSmmMpSyncData->CpuData[Index].Busy);\r
      ReleaseSpinLock (mSmmMpSyncData->CpuData[Index].Busy);\r
    }\r
  }\r
\r
  //\r
  // Perform the remaining tasks\r
  //\r
  PerformRemainingTasks ();\r
\r
  //\r
  // If Relaxed-AP Sync Mode: gather all available APs after BSP SMM handlers are done, and\r
  // make those APs to exit SMI synchronously. APs which arrive later will be excluded and\r
  // will run through freely.\r
  //\r
  if (SyncMode != SmmCpuSyncModeTradition && !SmmCpuFeaturesNeedConfigureMtrrs()) {\r
\r
    //\r
    // Lock the counter down and retrieve the number of APs\r
    //\r
    *mSmmMpSyncData->AllCpusInSync = TRUE;\r
    ApCount = LockdownSemaphore (mSmmMpSyncData->Counter) - 1;\r
    //\r
    // Make sure all APs have their Present flag set\r
    //\r
    while (TRUE) {\r
      PresentCount = 0;\r
      for (Index = mMaxNumberOfCpus; Index-- > 0;) {\r
        if (*(mSmmMpSyncData->CpuData[Index].Present)) {\r
          PresentCount ++;\r
        }\r
      }\r
      if (PresentCount > ApCount) {\r
        break;\r
      }\r
    }\r
  }\r
\r
  //\r
  // Notify all APs to exit\r
  //\r
  *mSmmMpSyncData->InsideSmm = FALSE;\r
  ReleaseAllAPs ();\r
\r
  //\r
  // Wait for all APs to complete their pending tasks\r
  //\r
  WaitForAllAPs (ApCount);\r
\r
  if (SmmCpuFeaturesNeedConfigureMtrrs()) {\r
    //\r
    // Signal APs to restore MTRRs\r
    //\r
    ReleaseAllAPs ();\r
\r
    //\r
    // Restore OS MTRRs\r
    //\r
    SmmCpuFeaturesReenableSmrr ();\r
    MtrrSetAllMtrrs(&Mtrrs);\r
\r
    //\r
    // Wait for all APs to complete MTRR programming\r
    //\r
    WaitForAllAPs (ApCount);\r
  }\r
\r
  //\r
  // Stop source level debug in BSP handler, the code below will not be\r
  // debugged.\r
  //\r
  InitializeDebugAgent (DEBUG_AGENT_INIT_EXIT_SMI, NULL, NULL);\r
\r
  //\r
  // Signal APs to Reset states/semaphore for this processor\r
  //\r
  ReleaseAllAPs ();\r
\r
  //\r
  // Perform pending operations for hot-plug\r
  //\r
  SmmCpuUpdate ();\r
\r
  //\r
  // Clear the Present flag of BSP\r
  //\r
  *(mSmmMpSyncData->CpuData[CpuIndex].Present) = FALSE;\r
\r
  //\r
  // Gather APs to exit SMM synchronously. Note the Present flag is cleared by now but\r
  // WaitForAllAps does not depend on the Present flag.\r
  //\r
  WaitForAllAPs (ApCount);\r
\r
  //\r
  // Reset BspIndex to -1, meaning BSP has not been elected.\r
  //\r
  if (FeaturePcdGet (PcdCpuSmmEnableBspElection)) {\r
    mSmmMpSyncData->BspIndex = (UINT32)-1;\r
  }\r
\r
  //\r
  // Allow APs to check in from this point on\r
  //\r
  *mSmmMpSyncData->Counter = 0;\r
  *mSmmMpSyncData->AllCpusInSync = FALSE;\r
}\r
\r
/**\r
  SMI handler for AP.\r
\r
  @param     CpuIndex         AP processor Index.\r
  @param     ValidSmi         Indicates that current SMI is a valid SMI or not.\r
  @param     SyncMode         SMM MP sync mode.\r
\r
**/\r
VOID\r
APHandler (\r
  IN      UINTN                     CpuIndex,\r
  IN      BOOLEAN                   ValidSmi,\r
  IN      SMM_CPU_SYNC_MODE         SyncMode\r
  )\r
{\r
  UINT64                            Timer;\r
  UINTN                             BspIndex;\r
  MTRR_SETTINGS                     Mtrrs;\r
\r
  //\r
  // Timeout BSP\r
  //\r
  for (Timer = StartSyncTimer ();\r
       !IsSyncTimerTimeout (Timer) &&\r
       !(*mSmmMpSyncData->InsideSmm);\r
       ) {\r
    CpuPause ();\r
  }\r
\r
  if (!(*mSmmMpSyncData->InsideSmm)) {\r
    //\r
    // BSP timeout in the first round\r
    //\r
    if (mSmmMpSyncData->BspIndex != -1) {\r
      //\r
      // BSP Index is known\r
      //\r
      BspIndex = mSmmMpSyncData->BspIndex;\r
      ASSERT (CpuIndex != BspIndex);\r
\r
      //\r
      // Send SMI IPI to bring BSP in\r
      //\r
      SendSmiIpi ((UINT32)gSmmCpuPrivate->ProcessorInfo[BspIndex].ProcessorId);\r
\r
      //\r
      // Now clock BSP for the 2nd time\r
      //\r
      for (Timer = StartSyncTimer ();\r
           !IsSyncTimerTimeout (Timer) &&\r
           !(*mSmmMpSyncData->InsideSmm);\r
           ) {\r
        CpuPause ();\r
      }\r
\r
      if (!(*mSmmMpSyncData->InsideSmm)) {\r
        //\r
        // Give up since BSP is unable to enter SMM\r
        // and signal the completion of this AP\r
        WaitForSemaphore (mSmmMpSyncData->Counter);\r
        return;\r
      }\r
    } else {\r
      //\r
      // Don't know BSP index. Give up without sending IPI to BSP.\r
      //\r
      WaitForSemaphore (mSmmMpSyncData->Counter);\r
      return;\r
    }\r
  }\r
\r
  //\r
  // BSP is available\r
  //\r
  BspIndex = mSmmMpSyncData->BspIndex;\r
  ASSERT (CpuIndex != BspIndex);\r
\r
  //\r
  // Mark this processor's presence\r
  //\r
  *(mSmmMpSyncData->CpuData[CpuIndex].Present) = TRUE;\r
\r
  if (SyncMode == SmmCpuSyncModeTradition || SmmCpuFeaturesNeedConfigureMtrrs()) {\r
    //\r
    // Notify BSP of arrival at this point\r
    //\r
    ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);\r
  }\r
\r
  if (SmmCpuFeaturesNeedConfigureMtrrs()) {\r
    //\r
    // Wait for the signal from BSP to backup MTRRs\r
    //\r
    WaitForSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);\r
\r
    //\r
    // Backup OS MTRRs\r
    //\r
    MtrrGetAllMtrrs(&Mtrrs);\r
\r
    //\r
    // Signal BSP the completion of this AP\r
    //\r
    ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);\r
\r
    //\r
    // Wait for BSP's signal to program MTRRs\r
    //\r
    WaitForSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);\r
\r
    //\r
    // Replace OS MTRRs with SMI MTRRs\r
    //\r
    ReplaceOSMtrrs (CpuIndex);\r
\r
    //\r
    // Signal BSP the completion of this AP\r
    //\r
    ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);\r
  }\r
\r
  while (TRUE) {\r
    //\r
    // Wait for something to happen\r
    //\r
    WaitForSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);\r
\r
    //\r
    // Check if BSP wants to exit SMM\r
    //\r
    if (!(*mSmmMpSyncData->InsideSmm)) {\r
      break;\r
    }\r
\r
    //\r
    // BUSY should be acquired by SmmStartupThisAp()\r
    //\r
    ASSERT (\r
      !AcquireSpinLockOrFail (mSmmMpSyncData->CpuData[CpuIndex].Busy)\r
      );\r
\r
    //\r
    // Invoke the scheduled procedure\r
    //\r
    (*mSmmMpSyncData->CpuData[CpuIndex].Procedure) (\r
      (VOID*)mSmmMpSyncData->CpuData[CpuIndex].Parameter\r
      );\r
\r
    //\r
    // Release BUSY\r
    //\r
    ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);\r
  }\r
\r
  if (SmmCpuFeaturesNeedConfigureMtrrs()) {\r
    //\r
    // Notify BSP the readiness of this AP to program MTRRs\r
    //\r
    ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);\r
\r
    //\r
    // Wait for the signal from BSP to program MTRRs\r
    //\r
    WaitForSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);\r
\r
    //\r
    // Restore OS MTRRs\r
    //\r
    SmmCpuFeaturesReenableSmrr ();\r
    MtrrSetAllMtrrs(&Mtrrs);\r
  }\r
\r
  //\r
  // Notify BSP the readiness of this AP to Reset states/semaphore for this processor\r
  //\r
  ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);\r
\r
  //\r
  // Wait for the signal from BSP to Reset states/semaphore for this processor\r
  //\r
  WaitForSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);\r
\r
  //\r
  // Reset states/semaphore for this processor\r
  //\r
  *(mSmmMpSyncData->CpuData[CpuIndex].Present) = FALSE;\r
\r
  //\r
  // Notify BSP the readiness of this AP to exit SMM\r
  //\r
  ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);\r
\r
}\r
\r
/**\r
  Create 4G PageTable in SMRAM.\r
\r
  @param[in]      Is32BitPageTable Whether the page table is 32-bit PAE\r
  @return         PageTable Address\r
\r
**/\r
UINT32\r
Gen4GPageTable (\r
  IN      BOOLEAN                   Is32BitPageTable\r
  )\r
{\r
  VOID    *PageTable;\r
  UINTN   Index;\r
  UINT64  *Pte;\r
  UINTN   PagesNeeded;\r
  UINTN   Low2MBoundary;\r
  UINTN   High2MBoundary;\r
  UINTN   Pages;\r
  UINTN   GuardPage;\r
  UINT64  *Pdpte;\r
  UINTN   PageIndex;\r
  UINTN   PageAddress;\r
\r
  Low2MBoundary = 0;\r
  High2MBoundary = 0;\r
  PagesNeeded = 0;\r
  if (FeaturePcdGet (PcdCpuSmmStackGuard)) {\r
    //\r
    // Add one more page for known good stack, then find the lower 2MB aligned address.\r
    //\r
    Low2MBoundary = (mSmmStackArrayBase + EFI_PAGE_SIZE) & ~(SIZE_2MB-1);\r
    //\r
    // Add two more pages for known good stack and stack guard page,\r
    // then find the lower 2MB aligned address.\r
    //\r
    High2MBoundary = (mSmmStackArrayEnd - mSmmStackSize + EFI_PAGE_SIZE * 2) & ~(SIZE_2MB-1);\r
    PagesNeeded = ((High2MBoundary - Low2MBoundary) / SIZE_2MB) + 1;\r
  }\r
  //\r
  // Allocate the page table\r
  //\r
  PageTable = AllocatePageTableMemory (5 + PagesNeeded);\r
  ASSERT (PageTable != NULL);\r
\r
  PageTable = (VOID *)((UINTN)PageTable);\r
  Pte = (UINT64*)PageTable;\r
\r
  //\r
  // Zero out all page table entries first\r
  //\r
  ZeroMem (Pte, EFI_PAGES_TO_SIZE (1));\r
\r
  //\r
  // Set Page Directory Pointers\r
  //\r
  for (Index = 0; Index < 4; Index++) {\r
    Pte[Index] = ((UINTN)PageTable + EFI_PAGE_SIZE * (Index + 1)) | mAddressEncMask |\r
                   (Is32BitPageTable ? IA32_PAE_PDPTE_ATTRIBUTE_BITS : PAGE_ATTRIBUTE_BITS);\r
  }\r
  Pte += EFI_PAGE_SIZE / sizeof (*Pte);\r
\r
  //\r
  // Fill in Page Directory Entries\r
  //\r
  for (Index = 0; Index < EFI_PAGE_SIZE * 4 / sizeof (*Pte); Index++) {\r
    Pte[Index] = (Index << 21) | mAddressEncMask | IA32_PG_PS | PAGE_ATTRIBUTE_BITS;\r
  }\r
\r
  Pdpte = (UINT64*)PageTable;\r
  if (FeaturePcdGet (PcdCpuSmmStackGuard)) {\r
    Pages = (UINTN)PageTable + EFI_PAGES_TO_SIZE (5);\r
    GuardPage = mSmmStackArrayBase + EFI_PAGE_SIZE;\r
    for (PageIndex = Low2MBoundary; PageIndex <= High2MBoundary; PageIndex += SIZE_2MB) {\r
      Pte = (UINT64*)(UINTN)(Pdpte[BitFieldRead32 ((UINT32)PageIndex, 30, 31)] & ~mAddressEncMask & ~(EFI_PAGE_SIZE - 1));\r
      Pte[BitFieldRead32 ((UINT32)PageIndex, 21, 29)] = (UINT64)Pages | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
      //\r
      // Fill in Page Table Entries\r
      //\r
      Pte = (UINT64*)Pages;\r
      PageAddress = PageIndex;\r
      for (Index = 0; Index < EFI_PAGE_SIZE / sizeof (*Pte); Index++) {\r
        if (PageAddress == GuardPage) {\r
          //\r
          // Mark the guard page as non-present\r
          //\r
          Pte[Index] = PageAddress | mAddressEncMask;\r
          GuardPage += mSmmStackSize;\r
          if (GuardPage > mSmmStackArrayEnd) {\r
            GuardPage = 0;\r
          }\r
        } else {\r
          Pte[Index] = PageAddress | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
        }\r
        PageAddress+= EFI_PAGE_SIZE;\r
      }\r
      Pages += EFI_PAGE_SIZE;\r
    }\r
  }\r
\r
  if ((PcdGet8 (PcdNullPointerDetectionPropertyMask) & BIT1) != 0) {\r
    Pte = (UINT64*)(UINTN)(Pdpte[0] & ~mAddressEncMask & ~(EFI_PAGE_SIZE - 1));\r
    if ((Pte[0] & IA32_PG_PS) == 0) {\r
      // 4K-page entries are already mapped. Just hide the first one anyway.\r
      Pte = (UINT64*)(UINTN)(Pte[0] & ~mAddressEncMask & ~(EFI_PAGE_SIZE - 1));\r
      Pte[0] &= ~(UINT64)IA32_PG_P; // Hide page 0\r
    } else {\r
      // Create 4K-page entries\r
      Pages = (UINTN)AllocatePageTableMemory (1);\r
      ASSERT (Pages != 0);\r
\r
      Pte[0] = (UINT64)(Pages | mAddressEncMask | PAGE_ATTRIBUTE_BITS);\r
\r
      Pte = (UINT64*)Pages;\r
      PageAddress = 0;\r
      Pte[0] = PageAddress | mAddressEncMask; // Hide page 0 but present left\r
      for (Index = 1; Index < EFI_PAGE_SIZE / sizeof (*Pte); Index++) {\r
        PageAddress += EFI_PAGE_SIZE;\r
        Pte[Index] = PageAddress | mAddressEncMask | PAGE_ATTRIBUTE_BITS;\r
      }\r
    }\r
  }\r
\r
  return (UINT32)(UINTN)PageTable;\r
}\r
\r
/**\r
  Schedule a procedure to run on the specified CPU.\r
\r
  @param[in]       Procedure                The address of the procedure to run\r
  @param[in]       CpuIndex                 Target CPU Index\r
  @param[in, out]  ProcArguments            The parameter to pass to the procedure\r
  @param[in]       BlockingMode             Startup AP in blocking mode or not\r
\r
  @retval EFI_INVALID_PARAMETER    CpuNumber not valid\r
  @retval EFI_INVALID_PARAMETER    CpuNumber specifying BSP\r
  @retval EFI_INVALID_PARAMETER    The AP specified by CpuNumber did not enter SMM\r
  @retval EFI_INVALID_PARAMETER    The AP specified by CpuNumber is busy\r
  @retval EFI_SUCCESS              The procedure has been successfully scheduled\r
\r
**/\r
EFI_STATUS\r
InternalSmmStartupThisAp (\r
  IN      EFI_AP_PROCEDURE          Procedure,\r
  IN      UINTN                     CpuIndex,\r
  IN OUT  VOID                      *ProcArguments OPTIONAL,\r
  IN      BOOLEAN                   BlockingMode\r
  )\r
{\r
  if (CpuIndex >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus) {\r
    DEBUG((DEBUG_ERROR, "CpuIndex(%d) >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus(%d)\n", CpuIndex, gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus));\r
    return EFI_INVALID_PARAMETER;\r
  }\r
  if (CpuIndex == gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu) {\r
    DEBUG((DEBUG_ERROR, "CpuIndex(%d) == gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu\n", CpuIndex));\r
    return EFI_INVALID_PARAMETER;\r
  }\r
  if (gSmmCpuPrivate->ProcessorInfo[CpuIndex].ProcessorId == INVALID_APIC_ID) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
  if (!(*(mSmmMpSyncData->CpuData[CpuIndex].Present))) {\r
    if (mSmmMpSyncData->EffectiveSyncMode == SmmCpuSyncModeTradition) {\r
      DEBUG((DEBUG_ERROR, "!mSmmMpSyncData->CpuData[%d].Present\n", CpuIndex));\r
    }\r
    return EFI_INVALID_PARAMETER;\r
  }\r
  if (gSmmCpuPrivate->Operation[CpuIndex] == SmmCpuRemove) {\r
    if (!FeaturePcdGet (PcdCpuHotPlugSupport)) {\r
      DEBUG((DEBUG_ERROR, "gSmmCpuPrivate->Operation[%d] == SmmCpuRemove\n", CpuIndex));\r
    }\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  if (BlockingMode) {\r
    AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);\r
  } else {\r
    if (!AcquireSpinLockOrFail (mSmmMpSyncData->CpuData[CpuIndex].Busy)) {\r
      DEBUG((DEBUG_ERROR, "mSmmMpSyncData->CpuData[%d].Busy\n", CpuIndex));\r
      return EFI_INVALID_PARAMETER;\r
    }\r
  }\r
\r
  mSmmMpSyncData->CpuData[CpuIndex].Procedure = Procedure;\r
  mSmmMpSyncData->CpuData[CpuIndex].Parameter = ProcArguments;\r
  ReleaseSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);\r
\r
  if (BlockingMode) {\r
    AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);\r
    ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);\r
  }\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Schedule a procedure to run on the specified CPU in blocking mode.\r
\r
  @param[in]       Procedure                The address of the procedure to run\r
  @param[in]       CpuIndex                 Target CPU Index\r
  @param[in, out]  ProcArguments            The parameter to pass to the procedure\r
\r
  @retval EFI_INVALID_PARAMETER    CpuNumber not valid\r
  @retval EFI_INVALID_PARAMETER    CpuNumber specifying BSP\r
  @retval EFI_INVALID_PARAMETER    The AP specified by CpuNumber did not enter SMM\r
  @retval EFI_INVALID_PARAMETER    The AP specified by CpuNumber is busy\r
  @retval EFI_SUCCESS              The procedure has been successfully scheduled\r
\r
**/\r
EFI_STATUS\r
EFIAPI\r
SmmBlockingStartupThisAp (\r
  IN      EFI_AP_PROCEDURE          Procedure,\r
  IN      UINTN                     CpuIndex,\r
  IN OUT  VOID                      *ProcArguments OPTIONAL\r
  )\r
{\r
  return InternalSmmStartupThisAp(Procedure, CpuIndex, ProcArguments, TRUE);\r
}\r
\r
/**\r
  Schedule a procedure to run on the specified CPU.\r
\r
  @param  Procedure                The address of the procedure to run\r
  @param  CpuIndex                 Target CPU Index\r
  @param  ProcArguments            The parameter to pass to the procedure\r
\r
  @retval EFI_INVALID_PARAMETER    CpuNumber not valid\r
  @retval EFI_INVALID_PARAMETER    CpuNumber specifying BSP\r
  @retval EFI_INVALID_PARAMETER    The AP specified by CpuNumber did not enter SMM\r
  @retval EFI_INVALID_PARAMETER    The AP specified by CpuNumber is busy\r
  @retval EFI_SUCCESS              The procedure has been successfully scheduled\r
\r
**/\r
EFI_STATUS\r
EFIAPI\r
SmmStartupThisAp (\r
  IN      EFI_AP_PROCEDURE          Procedure,\r
  IN      UINTN                     CpuIndex,\r
  IN OUT  VOID                      *ProcArguments OPTIONAL\r
  )\r
{\r
  return InternalSmmStartupThisAp(Procedure, CpuIndex, ProcArguments, FeaturePcdGet (PcdCpuSmmBlockStartupThisAp));\r
}\r
\r
/**\r
  This function sets DR6 & DR7 according to SMM save state, before running SMM C code.\r
  They are useful when you want to enable hardware breakpoints in SMM without entry SMM mode.\r
\r
  NOTE: It might not be appreciated in runtime since it might\r
        conflict with OS debugging facilities. Turn them off in RELEASE.\r
\r
  @param    CpuIndex              CPU Index\r
\r
**/\r
VOID\r
EFIAPI\r
CpuSmmDebugEntry (\r
  IN UINTN  CpuIndex\r
  )\r
{\r
  SMRAM_SAVE_STATE_MAP *CpuSaveState;\r
\r
  if (FeaturePcdGet (PcdCpuSmmDebug)) {\r
    ASSERT(CpuIndex < mMaxNumberOfCpus);\r
    CpuSaveState = (SMRAM_SAVE_STATE_MAP *)gSmmCpuPrivate->CpuSaveState[CpuIndex];\r
    if (mSmmSaveStateRegisterLma == EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT) {\r
      AsmWriteDr6 (CpuSaveState->x86._DR6);\r
      AsmWriteDr7 (CpuSaveState->x86._DR7);\r
    } else {\r
      AsmWriteDr6 ((UINTN)CpuSaveState->x64._DR6);\r
      AsmWriteDr7 ((UINTN)CpuSaveState->x64._DR7);\r
    }\r
  }\r
}\r
\r
/**\r
  This function restores DR6 & DR7 to SMM save state.\r
\r
  NOTE: It might not be appreciated in runtime since it might\r
        conflict with OS debugging facilities. Turn them off in RELEASE.\r
\r
  @param    CpuIndex              CPU Index\r
\r
**/\r
VOID\r
EFIAPI\r
CpuSmmDebugExit (\r
  IN UINTN  CpuIndex\r
  )\r
{\r
  SMRAM_SAVE_STATE_MAP *CpuSaveState;\r
\r
  if (FeaturePcdGet (PcdCpuSmmDebug)) {\r
    ASSERT(CpuIndex < mMaxNumberOfCpus);\r
    CpuSaveState = (SMRAM_SAVE_STATE_MAP *)gSmmCpuPrivate->CpuSaveState[CpuIndex];\r
    if (mSmmSaveStateRegisterLma == EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT) {\r
      CpuSaveState->x86._DR7 = (UINT32)AsmReadDr7 ();\r
      CpuSaveState->x86._DR6 = (UINT32)AsmReadDr6 ();\r
    } else {\r
      CpuSaveState->x64._DR7 = AsmReadDr7 ();\r
      CpuSaveState->x64._DR6 = AsmReadDr6 ();\r
    }\r
  }\r
}\r
\r
/**\r
  C function for SMI entry, each processor comes here upon SMI trigger.\r
\r
  @param    CpuIndex              CPU Index\r
\r
**/\r
VOID\r
EFIAPI\r
SmiRendezvous (\r
  IN      UINTN                     CpuIndex\r
  )\r
{\r
  EFI_STATUS                     Status;\r
  BOOLEAN                        ValidSmi;\r
  BOOLEAN                        IsBsp;\r
  BOOLEAN                        BspInProgress;\r
  UINTN                          Index;\r
  UINTN                          Cr2;\r
\r
  ASSERT(CpuIndex < mMaxNumberOfCpus);\r
\r
  //\r
  // Save Cr2 because Page Fault exception in SMM may override its value,\r
  // when using on-demand paging for above 4G memory.\r
  //\r
  Cr2 = 0;\r
  SaveCr2 (&Cr2);\r
\r
  //\r
  // Perform CPU specific entry hooks\r
  //\r
  SmmCpuFeaturesRendezvousEntry (CpuIndex);\r
\r
  //\r
  // Determine if this is a valid SMI\r
  //\r
  ValidSmi = PlatformValidSmi();\r
\r
  //\r
  // Determine if BSP has been already in progress. Note this must be checked after\r
  // ValidSmi because BSP may clear a valid SMI source after checking in.\r
  //\r
  BspInProgress = *mSmmMpSyncData->InsideSmm;\r
\r
  if (!BspInProgress && !ValidSmi) {\r
    //\r
    // If we reach here, it means when we sampled the ValidSmi flag, SMI status had not\r
    // been cleared by BSP in a new SMI run (so we have a truly invalid SMI), or SMI\r
    // status had been cleared by BSP and an existing SMI run has almost ended. (Note\r
    // we sampled ValidSmi flag BEFORE judging BSP-in-progress status.) In both cases, there\r
    // is nothing we need to do.\r
    //\r
    goto Exit;\r
  } else {\r
    //\r
    // Signal presence of this processor\r
    //\r
    if (ReleaseSemaphore (mSmmMpSyncData->Counter) == 0) {\r
      //\r
      // BSP has already ended the synchronization, so QUIT!!!\r
      //\r
\r
      //\r
      // Wait for BSP's signal to finish SMI\r
      //\r
      while (*mSmmMpSyncData->AllCpusInSync) {\r
        CpuPause ();\r
      }\r
      goto Exit;\r
    } else {\r
\r
      //\r
      // The BUSY lock is initialized to Released state.\r
      // This needs to be done early enough to be ready for BSP's SmmStartupThisAp() call.\r
      // E.g., with Relaxed AP flow, SmmStartupThisAp() may be called immediately\r
      // after AP's present flag is detected.\r
      //\r
      InitializeSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);\r
    }\r
\r
    if (FeaturePcdGet (PcdCpuSmmProfileEnable)) {\r
      ActivateSmmProfile (CpuIndex);\r
    }\r
\r
    if (BspInProgress) {\r
      //\r
      // BSP has been elected. Follow AP path, regardless of ValidSmi flag\r
      // as BSP may have cleared the SMI status\r
      //\r
      APHandler (CpuIndex, ValidSmi, mSmmMpSyncData->EffectiveSyncMode);\r
    } else {\r
      //\r
      // We have a valid SMI\r
      //\r
\r
      //\r
      // Elect BSP\r
      //\r
      IsBsp = FALSE;\r
      if (FeaturePcdGet (PcdCpuSmmEnableBspElection)) {\r
        if (!mSmmMpSyncData->SwitchBsp || mSmmMpSyncData->CandidateBsp[CpuIndex]) {\r
          //\r
          // Call platform hook to do BSP election\r
          //\r
          Status = PlatformSmmBspElection (&IsBsp);\r
          if (EFI_SUCCESS == Status) {\r
            //\r
            // Platform hook determines successfully\r
            //\r
            if (IsBsp) {\r
              mSmmMpSyncData->BspIndex = (UINT32)CpuIndex;\r
            }\r
          } else {\r
            //\r
            // Platform hook fails to determine, use default BSP election method\r
            //\r
            InterlockedCompareExchange32 (\r
              (UINT32*)&mSmmMpSyncData->BspIndex,\r
              (UINT32)-1,\r
              (UINT32)CpuIndex\r
              );\r
          }\r
        }\r
      }\r
\r
      //\r
      // "mSmmMpSyncData->BspIndex == CpuIndex" means this is the BSP\r
      //\r
      if (mSmmMpSyncData->BspIndex == CpuIndex) {\r
\r
        //\r
        // Clear last request for SwitchBsp.\r
        //\r
        if (mSmmMpSyncData->SwitchBsp) {\r
          mSmmMpSyncData->SwitchBsp = FALSE;\r
          for (Index = 0; Index < mMaxNumberOfCpus; Index++) {\r
            mSmmMpSyncData->CandidateBsp[Index] = FALSE;\r
          }\r
        }\r
\r
        if (FeaturePcdGet (PcdCpuSmmProfileEnable)) {\r
          SmmProfileRecordSmiNum ();\r
        }\r
\r
        //\r
        // BSP Handler is always called with a ValidSmi == TRUE\r
        //\r
        BSPHandler (CpuIndex, mSmmMpSyncData->EffectiveSyncMode);\r
      } else {\r
        APHandler (CpuIndex, ValidSmi, mSmmMpSyncData->EffectiveSyncMode);\r
      }\r
    }\r
\r
    ASSERT (*mSmmMpSyncData->CpuData[CpuIndex].Run == 0);\r
\r
    //\r
    // Wait for BSP's signal to exit SMI\r
    //\r
    while (*mSmmMpSyncData->AllCpusInSync) {\r
      CpuPause ();\r
    }\r
  }\r
\r
Exit:\r
  SmmCpuFeaturesRendezvousExit (CpuIndex);\r
\r
  //\r
  // Restore Cr2\r
  //\r
  RestoreCr2 (Cr2);\r
}\r
\r
/**\r
  Allocate buffer for all semaphores and spin locks.\r
\r
**/\r
VOID\r
InitializeSmmCpuSemaphores (\r
  VOID\r
  )\r
{\r
  UINTN                      ProcessorCount;\r
  UINTN                      TotalSize;\r
  UINTN                      GlobalSemaphoresSize;\r
  UINTN                      CpuSemaphoresSize;\r
  UINTN                      SemaphoreSize;\r
  UINTN                      Pages;\r
  UINTN                      *SemaphoreBlock;\r
  UINTN                      SemaphoreAddr;\r
\r
  SemaphoreSize   = GetSpinLockProperties ();\r
  ProcessorCount = gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus;\r
  GlobalSemaphoresSize = (sizeof (SMM_CPU_SEMAPHORE_GLOBAL) / sizeof (VOID *)) * SemaphoreSize;\r
  CpuSemaphoresSize    = (sizeof (SMM_CPU_SEMAPHORE_CPU) / sizeof (VOID *)) * ProcessorCount * SemaphoreSize;\r
  TotalSize = GlobalSemaphoresSize + CpuSemaphoresSize;\r
  DEBUG((EFI_D_INFO, "One Semaphore Size    = 0x%x\n", SemaphoreSize));\r
  DEBUG((EFI_D_INFO, "Total Semaphores Size = 0x%x\n", TotalSize));\r
  Pages = EFI_SIZE_TO_PAGES (TotalSize);\r
  SemaphoreBlock = AllocatePages (Pages);\r
  ASSERT (SemaphoreBlock != NULL);\r
  ZeroMem (SemaphoreBlock, TotalSize);\r
\r
  SemaphoreAddr = (UINTN)SemaphoreBlock;\r
  mSmmCpuSemaphores.SemaphoreGlobal.Counter       = (UINT32 *)SemaphoreAddr;\r
  SemaphoreAddr += SemaphoreSize;\r
  mSmmCpuSemaphores.SemaphoreGlobal.InsideSmm     = (BOOLEAN *)SemaphoreAddr;\r
  SemaphoreAddr += SemaphoreSize;\r
  mSmmCpuSemaphores.SemaphoreGlobal.AllCpusInSync = (BOOLEAN *)SemaphoreAddr;\r
  SemaphoreAddr += SemaphoreSize;\r
  mSmmCpuSemaphores.SemaphoreGlobal.PFLock        = (SPIN_LOCK *)SemaphoreAddr;\r
  SemaphoreAddr += SemaphoreSize;\r
  mSmmCpuSemaphores.SemaphoreGlobal.CodeAccessCheckLock\r
                                                  = (SPIN_LOCK *)SemaphoreAddr;\r
  SemaphoreAddr += SemaphoreSize;\r
\r
  SemaphoreAddr = (UINTN)SemaphoreBlock + GlobalSemaphoresSize;\r
  mSmmCpuSemaphores.SemaphoreCpu.Busy    = (SPIN_LOCK *)SemaphoreAddr;\r
  SemaphoreAddr += ProcessorCount * SemaphoreSize;\r
  mSmmCpuSemaphores.SemaphoreCpu.Run     = (UINT32 *)SemaphoreAddr;\r
  SemaphoreAddr += ProcessorCount * SemaphoreSize;\r
  mSmmCpuSemaphores.SemaphoreCpu.Present = (BOOLEAN *)SemaphoreAddr;\r
\r
  mPFLock                       = mSmmCpuSemaphores.SemaphoreGlobal.PFLock;\r
  mConfigSmmCodeAccessCheckLock = mSmmCpuSemaphores.SemaphoreGlobal.CodeAccessCheckLock;\r
\r
  mSemaphoreSize = SemaphoreSize;\r
}\r
\r
/**\r
  Initialize un-cacheable data.\r
\r
**/\r
VOID\r
EFIAPI\r
InitializeMpSyncData (\r
  VOID\r
  )\r
{\r
  UINTN                      CpuIndex;\r
\r
  if (mSmmMpSyncData != NULL) {\r
    //\r
    // mSmmMpSyncDataSize includes one structure of SMM_DISPATCHER_MP_SYNC_DATA, one\r
    // CpuData array of SMM_CPU_DATA_BLOCK and one CandidateBsp array of BOOLEAN.\r
    //\r
    ZeroMem (mSmmMpSyncData, mSmmMpSyncDataSize);\r
    mSmmMpSyncData->CpuData = (SMM_CPU_DATA_BLOCK *)((UINT8 *)mSmmMpSyncData + sizeof (SMM_DISPATCHER_MP_SYNC_DATA));\r
    mSmmMpSyncData->CandidateBsp = (BOOLEAN *)(mSmmMpSyncData->CpuData + gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus);\r
    if (FeaturePcdGet (PcdCpuSmmEnableBspElection)) {\r
      //\r
      // Enable BSP election by setting BspIndex to -1\r
      //\r
      mSmmMpSyncData->BspIndex = (UINT32)-1;\r
    }\r
    mSmmMpSyncData->EffectiveSyncMode = mCpuSmmSyncMode;\r
\r
    mSmmMpSyncData->Counter       = mSmmCpuSemaphores.SemaphoreGlobal.Counter;\r
    mSmmMpSyncData->InsideSmm     = mSmmCpuSemaphores.SemaphoreGlobal.InsideSmm;\r
    mSmmMpSyncData->AllCpusInSync = mSmmCpuSemaphores.SemaphoreGlobal.AllCpusInSync;\r
    ASSERT (mSmmMpSyncData->Counter != NULL && mSmmMpSyncData->InsideSmm != NULL &&\r
            mSmmMpSyncData->AllCpusInSync != NULL);\r
    *mSmmMpSyncData->Counter       = 0;\r
    *mSmmMpSyncData->InsideSmm     = FALSE;\r
    *mSmmMpSyncData->AllCpusInSync = FALSE;\r
\r
    for (CpuIndex = 0; CpuIndex < gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus; CpuIndex ++) {\r
      mSmmMpSyncData->CpuData[CpuIndex].Busy    =\r
        (SPIN_LOCK *)((UINTN)mSmmCpuSemaphores.SemaphoreCpu.Busy + mSemaphoreSize * CpuIndex);\r
      mSmmMpSyncData->CpuData[CpuIndex].Run     =\r
        (UINT32 *)((UINTN)mSmmCpuSemaphores.SemaphoreCpu.Run + mSemaphoreSize * CpuIndex);\r
      mSmmMpSyncData->CpuData[CpuIndex].Present =\r
        (BOOLEAN *)((UINTN)mSmmCpuSemaphores.SemaphoreCpu.Present + mSemaphoreSize * CpuIndex);\r
      *(mSmmMpSyncData->CpuData[CpuIndex].Busy)    = 0;\r
      *(mSmmMpSyncData->CpuData[CpuIndex].Run)     = 0;\r
      *(mSmmMpSyncData->CpuData[CpuIndex].Present) = FALSE;\r
    }\r
  }\r
}\r
\r
/**\r
  Initialize global data for MP synchronization.\r
\r
  @param Stacks             Base address of SMI stack buffer for all processors.\r
  @param StackSize          Stack size for each processor in SMM.\r
  @param ShadowStackSize    Shadow Stack size for each processor in SMM.\r
\r
**/\r
UINT32\r
InitializeMpServiceData (\r
  IN VOID        *Stacks,\r
  IN UINTN       StackSize,\r
  IN UINTN       ShadowStackSize\r
  )\r
{\r
  UINT32                    Cr3;\r
  UINTN                     Index;\r
  UINT8                     *GdtTssTables;\r
  UINTN                     GdtTableStepSize;\r
  CPUID_VERSION_INFO_EDX    RegEdx;\r
\r
  //\r
  // Determine if this CPU supports machine check\r
  //\r
  AsmCpuid (CPUID_VERSION_INFO, NULL, NULL, NULL, &RegEdx.Uint32);\r
  mMachineCheckSupported = (BOOLEAN)(RegEdx.Bits.MCA == 1);\r
\r
  //\r
  // Allocate memory for all locks and semaphores\r
  //\r
  InitializeSmmCpuSemaphores ();\r
\r
  //\r
  // Initialize mSmmMpSyncData\r
  //\r
  mSmmMpSyncDataSize = sizeof (SMM_DISPATCHER_MP_SYNC_DATA) +\r
                       (sizeof (SMM_CPU_DATA_BLOCK) + sizeof (BOOLEAN)) * gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus;\r
  mSmmMpSyncData = (SMM_DISPATCHER_MP_SYNC_DATA*) AllocatePages (EFI_SIZE_TO_PAGES (mSmmMpSyncDataSize));\r
  ASSERT (mSmmMpSyncData != NULL);\r
  mCpuSmmSyncMode = (SMM_CPU_SYNC_MODE)PcdGet8 (PcdCpuSmmSyncMode);\r
  InitializeMpSyncData ();\r
\r
  //\r
  // Initialize physical address mask\r
  // NOTE: Physical memory above virtual address limit is not supported !!!\r
  //\r
  AsmCpuid (0x80000008, (UINT32*)&Index, NULL, NULL, NULL);\r
  gPhyMask = LShiftU64 (1, (UINT8)Index) - 1;\r
  gPhyMask &= (1ull << 48) - EFI_PAGE_SIZE;\r
\r
  //\r
  // Create page tables\r
  //\r
  Cr3 = SmmInitPageTable ();\r
\r
  GdtTssTables = InitGdt (Cr3, &GdtTableStepSize);\r
\r
  //\r
  // Install SMI handler for each CPU\r
  //\r
  for (Index = 0; Index < mMaxNumberOfCpus; Index++) {\r
    InstallSmiHandler (\r
      Index,\r
      (UINT32)mCpuHotPlugData.SmBase[Index],\r
      (VOID*)((UINTN)Stacks + (StackSize + ShadowStackSize) * Index),\r
      StackSize,\r
      (UINTN)(GdtTssTables + GdtTableStepSize * Index),\r
      gcSmiGdtr.Limit + 1,\r
      gcSmiIdtr.Base,\r
      gcSmiIdtr.Limit + 1,\r
      Cr3\r
      );\r
  }\r
\r
  //\r
  // Record current MTRR settings\r
  //\r
  ZeroMem (&gSmiMtrrs, sizeof (gSmiMtrrs));\r
  MtrrGetAllMtrrs (&gSmiMtrrs);\r
\r
  return Cr3;\r
}\r
\r
/**\r
\r
  Register the SMM Foundation entry point.\r
\r
  @param          This              Pointer to EFI_SMM_CONFIGURATION_PROTOCOL instance\r
  @param          SmmEntryPoint     SMM Foundation EntryPoint\r
\r
  @retval         EFI_SUCCESS       Successfully to register SMM foundation entry point\r
\r
**/\r
EFI_STATUS\r
EFIAPI\r
RegisterSmmEntry (\r
  IN CONST EFI_SMM_CONFIGURATION_PROTOCOL  *This,\r
  IN EFI_SMM_ENTRY_POINT                   SmmEntryPoint\r
  )\r
{\r
  //\r
  // Record SMM Foundation EntryPoint, later invoke it on SMI entry vector.\r
  //\r
  gSmmCpuPrivate->SmmCoreEntry = SmmEntryPoint;\r
  return EFI_SUCCESS;\r
}\r