X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=UefiCpuPkg%2FPiSmmCpuDxeSmm%2FMpService.c;h=870250b0c55d0209c113c9cd8a6b2328d81e8e06;hb=b948a496150f4ae4f656c0f0ab672608723c80e6;hp=3e8b0936aa8415aa1f980a77ccbd399eaff6dd78;hpb=9f419739d1ae849e0c4d75a131502f9367ca4a7d;p=mirror_edk2.git
diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
index 3e8b0936aa..870250b0c5 100644
--- a/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
+++ b/UefiCpuPkg/PiSmmCpuDxeSmm/MpService.c
@@ -1,14 +1,10 @@
/** @file
SMM MP service implementation
-Copyright (c) 2009 - 2015, Intel Corporation. All rights reserved.
-This program and the accompanying materials
-are licensed and made available under the terms and conditions of the BSD License
-which accompanies this distribution. The full text of the license may be found at
-http://opensource.org/licenses/bsd-license.php
+Copyright (c) 2009 - 2019, Intel Corporation. All rights reserved.
+Copyright (c) 2017, AMD Incorporated. All rights reserved.
-THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
-WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+SPDX-License-Identifier: BSD-2-Clause-Patent
**/
@@ -17,10 +13,15 @@ WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
//
// Slots for all MTRR( FIXED MTRR + VARIABLE MTRR + MTRR_LIB_IA32_MTRR_DEF_TYPE)
//
-UINT64 gSmiMtrrs[MTRR_NUMBER_OF_FIXED_MTRR + 2 * MTRR_NUMBER_OF_VARIABLE_MTRR + 1];
+MTRR_SETTINGS gSmiMtrrs;
UINT64 gPhyMask;
SMM_DISPATCHER_MP_SYNC_DATA *mSmmMpSyncData = NULL;
UINTN mSmmMpSyncDataSize;
+SMM_CPU_SEMAPHORES mSmmCpuSemaphores;
+UINTN mSemaphoreSize;
+SPIN_LOCK *mPFLock = NULL;
+SMM_CPU_SYNC_MODE mCpuSmmSyncMode;
+BOOLEAN mMachineCheckSupported = FALSE;
/**
Performs an atomic compare exchange operation to get semaphore.
@@ -120,7 +121,7 @@ WaitForAllAPs (
BspIndex = mSmmMpSyncData->BspIndex;
while (NumberOfAPs-- > 0) {
- WaitForSemaphore (&mSmmMpSyncData->CpuData[BspIndex].Run);
+ WaitForSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);
}
}
@@ -135,12 +136,10 @@ ReleaseAllAPs (
)
{
UINTN Index;
- UINTN BspIndex;
- BspIndex = mSmmMpSyncData->BspIndex;
- for (Index = mMaxNumberOfCpus; Index-- > 0;) {
- if (Index != BspIndex && mSmmMpSyncData->CpuData[Index].Present) {
- ReleaseSemaphore (&mSmmMpSyncData->CpuData[Index].Run);
+ for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
+ if (IsPresentAp (Index)) {
+ ReleaseSemaphore (mSmmMpSyncData->CpuData[Index].Run);
}
}
}
@@ -163,16 +162,16 @@ AllCpusInSmmWithExceptions (
SMM_CPU_DATA_BLOCK *CpuData;
EFI_PROCESSOR_INFORMATION *ProcessorInfo;
- ASSERT (mSmmMpSyncData->Counter <= mNumberOfCpus);
+ ASSERT (*mSmmMpSyncData->Counter <= mNumberOfCpus);
- if (mSmmMpSyncData->Counter == mNumberOfCpus) {
+ if (*mSmmMpSyncData->Counter == mNumberOfCpus) {
return TRUE;
}
CpuData = mSmmMpSyncData->CpuData;
ProcessorInfo = gSmmCpuPrivate->ProcessorInfo;
- for (Index = mMaxNumberOfCpus; Index-- > 0;) {
- if (!CpuData[Index].Present && ProcessorInfo[Index].ProcessorId != INVALID_APIC_ID) {
+ for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
+ if (!(*(CpuData[Index].Present)) && ProcessorInfo[Index].ProcessorId != INVALID_APIC_ID) {
if (((Exceptions & ARRIVAL_EXCEPTION_DELAYED) != 0) && SmmCpuFeaturesGetSmmRegister (Index, SmmRegSmmDelayed) != 0) {
continue;
}
@@ -190,6 +189,56 @@ AllCpusInSmmWithExceptions (
return TRUE;
}
+/**
+ Has OS enabled Lmce in the MSR_IA32_MCG_EXT_CTL
+
+ @retval TRUE Os enable lmce.
+ @retval FALSE Os not enable lmce.
+
+**/
+BOOLEAN
+IsLmceOsEnabled (
+ VOID
+ )
+{
+ MSR_IA32_MCG_CAP_REGISTER McgCap;
+ MSR_IA32_FEATURE_CONTROL_REGISTER FeatureCtrl;
+ MSR_IA32_MCG_EXT_CTL_REGISTER McgExtCtrl;
+
+ McgCap.Uint64 = AsmReadMsr64 (MSR_IA32_MCG_CAP);
+ if (McgCap.Bits.MCG_LMCE_P == 0) {
+ return FALSE;
+ }
+
+ FeatureCtrl.Uint64 = AsmReadMsr64 (MSR_IA32_FEATURE_CONTROL);
+ if (FeatureCtrl.Bits.LmceOn == 0) {
+ return FALSE;
+ }
+
+ McgExtCtrl.Uint64 = AsmReadMsr64 (MSR_IA32_MCG_EXT_CTL);
+ return (BOOLEAN) (McgExtCtrl.Bits.LMCE_EN == 1);
+}
+
+/**
+ Return if Local machine check exception signaled.
+
+ Indicates (when set) that a local machine check exception was generated. This indicates that the current machine-check event was
+ delivered to only the logical processor.
+
+ @retval TRUE LMCE was signaled.
+ @retval FALSE LMCE was not signaled.
+
+**/
+BOOLEAN
+IsLmceSignaled (
+ VOID
+ )
+{
+ MSR_IA32_MCG_STATUS_REGISTER McgStatus;
+
+ McgStatus.Uint64 = AsmReadMsr64 (MSR_IA32_MCG_STATUS);
+ return (BOOLEAN) (McgStatus.Bits.LMCE_S == 1);
+}
/**
Given timeout constraint, wait for all APs to arrive, and insure when this function returns, no AP will execute normal mode code before
@@ -203,8 +252,17 @@ SmmWaitForApArrival (
{
UINT64 Timer;
UINTN Index;
+ BOOLEAN LmceEn;
+ BOOLEAN LmceSignal;
- ASSERT (mSmmMpSyncData->Counter <= mNumberOfCpus);
+ ASSERT (*mSmmMpSyncData->Counter <= mNumberOfCpus);
+
+ LmceEn = FALSE;
+ LmceSignal = FALSE;
+ if (mMachineCheckSupported) {
+ LmceEn = IsLmceOsEnabled ();
+ LmceSignal = IsLmceSignaled();
+ }
//
// Platform implementor should choose a timeout value appropriately:
@@ -221,7 +279,7 @@ SmmWaitForApArrival (
// Sync with APs 1st timeout
//
for (Timer = StartSyncTimer ();
- !IsSyncTimerTimeout (Timer) &&
+ !IsSyncTimerTimeout (Timer) && !(LmceEn && LmceSignal) &&
!AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED | ARRIVAL_EXCEPTION_SMI_DISABLED );
) {
CpuPause ();
@@ -243,12 +301,12 @@ SmmWaitForApArrival (
// - In relaxed flow, CheckApArrival() will check SMI disabling status before calling this function.
// In both cases, adding SMI-disabling checking code increases overhead.
//
- if (mSmmMpSyncData->Counter < mNumberOfCpus) {
+ if (*mSmmMpSyncData->Counter < mNumberOfCpus) {
//
// Send SMI IPIs to bring outside processors in
//
- for (Index = mMaxNumberOfCpus; Index-- > 0;) {
- if (!mSmmMpSyncData->CpuData[Index].Present && gSmmCpuPrivate->ProcessorInfo[Index].ProcessorId != INVALID_APIC_ID) {
+ for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
+ if (!(*(mSmmMpSyncData->CpuData[Index].Present)) && gSmmCpuPrivate->ProcessorInfo[Index].ProcessorId != INVALID_APIC_ID) {
SendSmiIpi ((UINT32)gSmmCpuPrivate->ProcessorInfo[Index].ProcessorId);
}
}
@@ -279,20 +337,122 @@ ReplaceOSMtrrs (
IN UINTN CpuIndex
)
{
- PROCESSOR_SMM_DESCRIPTOR *Psd;
- UINT64 *SmiMtrrs;
- MTRR_SETTINGS *BiosMtrr;
-
- Psd = (PROCESSOR_SMM_DESCRIPTOR*)(mCpuHotPlugData.SmBase[CpuIndex] + SMM_PSD_OFFSET);
- SmiMtrrs = (UINT64*)(UINTN)Psd->MtrrBaseMaskPtr;
-
SmmCpuFeaturesDisableSmrr ();
//
// Replace all MTRRs registers
//
- BiosMtrr = (MTRR_SETTINGS*)SmiMtrrs;
- MtrrSetAllMtrrs(BiosMtrr);
+ MtrrSetAllMtrrs (&gSmiMtrrs);
+}
+
+/**
+ Wheck whether task has been finished by all APs.
+
+ @param BlockMode Whether did it in block mode or non-block mode.
+
+ @retval TRUE Task has been finished by all APs.
+ @retval FALSE Task not has been finished by all APs.
+
+**/
+BOOLEAN
+WaitForAllAPsNotBusy (
+ IN BOOLEAN BlockMode
+ )
+{
+ UINTN Index;
+
+ for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
+ //
+ // Ignore BSP and APs which not call in SMM.
+ //
+ if (!IsPresentAp(Index)) {
+ continue;
+ }
+
+ if (BlockMode) {
+ AcquireSpinLock(mSmmMpSyncData->CpuData[Index].Busy);
+ ReleaseSpinLock(mSmmMpSyncData->CpuData[Index].Busy);
+ } else {
+ if (AcquireSpinLockOrFail (mSmmMpSyncData->CpuData[Index].Busy)) {
+ ReleaseSpinLock(mSmmMpSyncData->CpuData[Index].Busy);
+ } else {
+ return FALSE;
+ }
+ }
+ }
+
+ return TRUE;
+}
+
+/**
+ Check whether it is an present AP.
+
+ @param CpuIndex The AP index which calls this function.
+
+ @retval TRUE It's a present AP.
+ @retval TRUE This is not an AP or it is not present.
+
+**/
+BOOLEAN
+IsPresentAp (
+ IN UINTN CpuIndex
+ )
+{
+ return ((CpuIndex != gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu) &&
+ *(mSmmMpSyncData->CpuData[CpuIndex].Present));
+}
+
+/**
+ Clean up the status flags used during executing the procedure.
+
+ @param CpuIndex The AP index which calls this function.
+
+**/
+VOID
+ReleaseToken (
+ IN UINTN CpuIndex
+ )
+{
+ PROCEDURE_TOKEN *Token;
+
+ Token = mSmmMpSyncData->CpuData[CpuIndex].Token;
+
+ if (InterlockedDecrement (&Token->RunningApCount) == 0) {
+ ReleaseSpinLock (Token->SpinLock);
+ }
+
+ mSmmMpSyncData->CpuData[CpuIndex].Token = NULL;
+}
+
+/**
+ Free the tokens in the maintained list.
+
+**/
+VOID
+ResetTokens (
+ VOID
+ )
+{
+ LIST_ENTRY *Link;
+ PROCEDURE_TOKEN *ProcToken;
+
+ Link = GetFirstNode (&gSmmCpuPrivate->TokenList);
+ while (!IsNull (&gSmmCpuPrivate->TokenList, Link)) {
+ ProcToken = PROCEDURE_TOKEN_FROM_LINK (Link);
+
+ ProcToken->RunningApCount = 0;
+ ProcToken->Used = FALSE;
+
+ //
+ // Check the spinlock status and release it if not released yet.
+ //
+ if (!AcquireSpinLockOrFail(ProcToken->SpinLock)) {
+ DEBUG((DEBUG_ERROR, "Risk::SpinLock still not released!"));
+ }
+ ReleaseSpinLock (ProcToken->SpinLock);
+
+ Link = GetNextNode (&gSmmCpuPrivate->TokenList, Link);
+ }
}
/**
@@ -320,7 +480,7 @@ BSPHandler (
//
// Flag BSP's presence
//
- mSmmMpSyncData->InsideSmm = TRUE;
+ *mSmmMpSyncData->InsideSmm = TRUE;
//
// Initialize Debug Agent to start source level debug in BSP handler
@@ -330,7 +490,7 @@ BSPHandler (
//
// Mark this processor's presence
//
- mSmmMpSyncData->CpuData[CpuIndex].Present = TRUE;
+ *(mSmmMpSyncData->CpuData[CpuIndex].Present) = TRUE;
//
// Clear platform top level SMI status bit before calling SMI handlers. If
@@ -358,8 +518,8 @@ BSPHandler (
//
// Lock the counter down and retrieve the number of APs
//
- mSmmMpSyncData->AllCpusInSync = TRUE;
- ApCount = LockdownSemaphore (&mSmmMpSyncData->Counter) - 1;
+ *mSmmMpSyncData->AllCpusInSync = TRUE;
+ ApCount = LockdownSemaphore (mSmmMpSyncData->Counter) - 1;
//
// Wait for all APs to get ready for programming MTRRs
@@ -409,7 +569,7 @@ BSPHandler (
//
// The BUSY lock is initialized to Acquired state
//
- AcquireSpinLockOrFail (&mSmmMpSyncData->CpuData[CpuIndex].Busy);
+ AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
//
// Perform the pre tasks
@@ -424,12 +584,7 @@ BSPHandler (
//
// Make sure all APs have completed their pending none-block tasks
//
- for (Index = mMaxNumberOfCpus; Index-- > 0;) {
- if (Index != CpuIndex && mSmmMpSyncData->CpuData[Index].Present) {
- AcquireSpinLock (&mSmmMpSyncData->CpuData[Index].Busy);
- ReleaseSpinLock (&mSmmMpSyncData->CpuData[Index].Busy);;
- }
- }
+ WaitForAllAPsNotBusy (TRUE);
//
// Perform the remaining tasks
@@ -446,15 +601,15 @@ BSPHandler (
//
// Lock the counter down and retrieve the number of APs
//
- mSmmMpSyncData->AllCpusInSync = TRUE;
- ApCount = LockdownSemaphore (&mSmmMpSyncData->Counter) - 1;
+ *mSmmMpSyncData->AllCpusInSync = TRUE;
+ ApCount = LockdownSemaphore (mSmmMpSyncData->Counter) - 1;
//
// Make sure all APs have their Present flag set
//
while (TRUE) {
PresentCount = 0;
- for (Index = mMaxNumberOfCpus; Index-- > 0;) {
- if (mSmmMpSyncData->CpuData[Index].Present) {
+ for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
+ if (*(mSmmMpSyncData->CpuData[Index].Present)) {
PresentCount ++;
}
}
@@ -467,7 +622,7 @@ BSPHandler (
//
// Notify all APs to exit
//
- mSmmMpSyncData->InsideSmm = FALSE;
+ *mSmmMpSyncData->InsideSmm = FALSE;
ReleaseAllAPs ();
//
@@ -512,7 +667,7 @@ BSPHandler (
//
// Clear the Present flag of BSP
//
- mSmmMpSyncData->CpuData[CpuIndex].Present = FALSE;
+ *(mSmmMpSyncData->CpuData[CpuIndex].Present) = FALSE;
//
// Gather APs to exit SMM synchronously. Note the Present flag is cleared by now but
@@ -520,6 +675,11 @@ BSPHandler (
//
WaitForAllAPs (ApCount);
+ //
+ // Reset the tokens buffer.
+ //
+ ResetTokens ();
+
//
// Reset BspIndex to -1, meaning BSP has not been elected.
//
@@ -530,8 +690,8 @@ BSPHandler (
//
// Allow APs to check in from this point on
//
- mSmmMpSyncData->Counter = 0;
- mSmmMpSyncData->AllCpusInSync = FALSE;
+ *mSmmMpSyncData->Counter = 0;
+ *mSmmMpSyncData->AllCpusInSync = FALSE;
}
/**
@@ -552,18 +712,19 @@ APHandler (
UINT64 Timer;
UINTN BspIndex;
MTRR_SETTINGS Mtrrs;
+ EFI_STATUS ProcedureStatus;
//
// Timeout BSP
//
for (Timer = StartSyncTimer ();
!IsSyncTimerTimeout (Timer) &&
- !mSmmMpSyncData->InsideSmm;
+ !(*mSmmMpSyncData->InsideSmm);
) {
CpuPause ();
}
- if (!mSmmMpSyncData->InsideSmm) {
+ if (!(*mSmmMpSyncData->InsideSmm)) {
//
// BSP timeout in the first round
//
@@ -584,23 +745,23 @@ APHandler (
//
for (Timer = StartSyncTimer ();
!IsSyncTimerTimeout (Timer) &&
- !mSmmMpSyncData->InsideSmm;
+ !(*mSmmMpSyncData->InsideSmm);
) {
CpuPause ();
}
- if (!mSmmMpSyncData->InsideSmm) {
+ if (!(*mSmmMpSyncData->InsideSmm)) {
//
// Give up since BSP is unable to enter SMM
// and signal the completion of this AP
- WaitForSemaphore (&mSmmMpSyncData->Counter);
+ WaitForSemaphore (mSmmMpSyncData->Counter);
return;
}
} else {
//
// Don't know BSP index. Give up without sending IPI to BSP.
//
- WaitForSemaphore (&mSmmMpSyncData->Counter);
+ WaitForSemaphore (mSmmMpSyncData->Counter);
return;
}
}
@@ -614,20 +775,20 @@ APHandler (
//
// Mark this processor's presence
//
- mSmmMpSyncData->CpuData[CpuIndex].Present = TRUE;
+ *(mSmmMpSyncData->CpuData[CpuIndex].Present) = TRUE;
if (SyncMode == SmmCpuSyncModeTradition || SmmCpuFeaturesNeedConfigureMtrrs()) {
//
// Notify BSP of arrival at this point
//
- ReleaseSemaphore (&mSmmMpSyncData->CpuData[BspIndex].Run);
+ ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);
}
if (SmmCpuFeaturesNeedConfigureMtrrs()) {
//
// Wait for the signal from BSP to backup MTRRs
//
- WaitForSemaphore (&mSmmMpSyncData->CpuData[CpuIndex].Run);
+ WaitForSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);
//
// Backup OS MTRRs
@@ -637,12 +798,12 @@ APHandler (
//
// Signal BSP the completion of this AP
//
- ReleaseSemaphore (&mSmmMpSyncData->CpuData[BspIndex].Run);
+ ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);
//
// Wait for BSP's signal to program MTRRs
//
- WaitForSemaphore (&mSmmMpSyncData->CpuData[CpuIndex].Run);
+ WaitForSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);
//
// Replace OS MTRRs with SMI MTRRs
@@ -652,19 +813,19 @@ APHandler (
//
// Signal BSP the completion of this AP
//
- ReleaseSemaphore (&mSmmMpSyncData->CpuData[BspIndex].Run);
+ ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);
}
while (TRUE) {
//
// Wait for something to happen
//
- WaitForSemaphore (&mSmmMpSyncData->CpuData[CpuIndex].Run);
+ WaitForSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);
//
// Check if BSP wants to exit SMM
//
- if (!mSmmMpSyncData->InsideSmm) {
+ if (!(*mSmmMpSyncData->InsideSmm)) {
break;
}
@@ -672,32 +833,39 @@ APHandler (
// BUSY should be acquired by SmmStartupThisAp()
//
ASSERT (
- !AcquireSpinLockOrFail (&mSmmMpSyncData->CpuData[CpuIndex].Busy)
+ !AcquireSpinLockOrFail (mSmmMpSyncData->CpuData[CpuIndex].Busy)
);
//
// Invoke the scheduled procedure
//
- (*mSmmMpSyncData->CpuData[CpuIndex].Procedure) (
- (VOID*)mSmmMpSyncData->CpuData[CpuIndex].Parameter
- );
+ ProcedureStatus = (*mSmmMpSyncData->CpuData[CpuIndex].Procedure) (
+ (VOID*)mSmmMpSyncData->CpuData[CpuIndex].Parameter
+ );
+ if (mSmmMpSyncData->CpuData[CpuIndex].Status != NULL) {
+ *mSmmMpSyncData->CpuData[CpuIndex].Status = ProcedureStatus;
+ }
+
+ if (mSmmMpSyncData->CpuData[CpuIndex].Token != NULL) {
+ ReleaseToken (CpuIndex);
+ }
//
// Release BUSY
//
- ReleaseSpinLock (&mSmmMpSyncData->CpuData[CpuIndex].Busy);
+ ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
}
if (SmmCpuFeaturesNeedConfigureMtrrs()) {
//
// Notify BSP the readiness of this AP to program MTRRs
//
- ReleaseSemaphore (&mSmmMpSyncData->CpuData[BspIndex].Run);
+ ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);
//
// Wait for the signal from BSP to program MTRRs
//
- WaitForSemaphore (&mSmmMpSyncData->CpuData[CpuIndex].Run);
+ WaitForSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);
//
// Restore OS MTRRs
@@ -709,35 +877,35 @@ APHandler (
//
// Notify BSP the readiness of this AP to Reset states/semaphore for this processor
//
- ReleaseSemaphore (&mSmmMpSyncData->CpuData[BspIndex].Run);
+ ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);
//
// Wait for the signal from BSP to Reset states/semaphore for this processor
//
- WaitForSemaphore (&mSmmMpSyncData->CpuData[CpuIndex].Run);
+ WaitForSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);
//
// Reset states/semaphore for this processor
//
- mSmmMpSyncData->CpuData[CpuIndex].Present = FALSE;
+ *(mSmmMpSyncData->CpuData[CpuIndex].Present) = FALSE;
//
// Notify BSP the readiness of this AP to exit SMM
//
- ReleaseSemaphore (&mSmmMpSyncData->CpuData[BspIndex].Run);
+ ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);
}
/**
Create 4G PageTable in SMRAM.
- @param ExtraPages Additional page numbers besides for 4G memory
+ @param[in] Is32BitPageTable Whether the page table is 32-bit PAE
@return PageTable Address
**/
UINT32
Gen4GPageTable (
- IN UINTN ExtraPages
+ IN BOOLEAN Is32BitPageTable
)
{
VOID *PageTable;
@@ -770,10 +938,10 @@ Gen4GPageTable (
//
// Allocate the page table
//
- PageTable = AllocatePages (ExtraPages + 5 + PagesNeeded);
+ PageTable = AllocatePageTableMemory (5 + PagesNeeded);
ASSERT (PageTable != NULL);
- PageTable = (VOID *)((UINTN)PageTable + EFI_PAGES_TO_SIZE (ExtraPages));
+ PageTable = (VOID *)((UINTN)PageTable);
Pte = (UINT64*)PageTable;
//
@@ -785,7 +953,8 @@ Gen4GPageTable (
// Set Page Directory Pointers
//
for (Index = 0; Index < 4; Index++) {
- Pte[Index] = (UINTN)PageTable + EFI_PAGE_SIZE * (Index + 1) + IA32_PG_P;
+ Pte[Index] = ((UINTN)PageTable + EFI_PAGE_SIZE * (Index + 1)) | mAddressEncMask |
+ (Is32BitPageTable ? IA32_PAE_PDPTE_ATTRIBUTE_BITS : PAGE_ATTRIBUTE_BITS);
}
Pte += EFI_PAGE_SIZE / sizeof (*Pte);
@@ -793,16 +962,16 @@ Gen4GPageTable (
// Fill in Page Directory Entries
//
for (Index = 0; Index < EFI_PAGE_SIZE * 4 / sizeof (*Pte); Index++) {
- Pte[Index] = (Index << 21) + IA32_PG_PS + IA32_PG_RW + IA32_PG_P;
+ Pte[Index] = (Index << 21) | mAddressEncMask | IA32_PG_PS | PAGE_ATTRIBUTE_BITS;
}
+ Pdpte = (UINT64*)PageTable;
if (FeaturePcdGet (PcdCpuSmmStackGuard)) {
Pages = (UINTN)PageTable + EFI_PAGES_TO_SIZE (5);
GuardPage = mSmmStackArrayBase + EFI_PAGE_SIZE;
- Pdpte = (UINT64*)PageTable;
for (PageIndex = Low2MBoundary; PageIndex <= High2MBoundary; PageIndex += SIZE_2MB) {
- Pte = (UINT64*)(UINTN)(Pdpte[BitFieldRead32 ((UINT32)PageIndex, 30, 31)] & ~(EFI_PAGE_SIZE - 1));
- Pte[BitFieldRead32 ((UINT32)PageIndex, 21, 29)] = (UINT64)Pages + IA32_PG_RW + IA32_PG_P;
+ Pte = (UINT64*)(UINTN)(Pdpte[BitFieldRead32 ((UINT32)PageIndex, 30, 31)] & ~mAddressEncMask & ~(EFI_PAGE_SIZE - 1));
+ Pte[BitFieldRead32 ((UINT32)PageIndex, 21, 29)] = (UINT64)Pages | mAddressEncMask | PAGE_ATTRIBUTE_BITS;
//
// Fill in Page Table Entries
//
@@ -813,13 +982,13 @@ Gen4GPageTable (
//
// Mark the guard page as non-present
//
- Pte[Index] = PageAddress;
+ Pte[Index] = PageAddress | mAddressEncMask;
GuardPage += mSmmStackSize;
if (GuardPage > mSmmStackArrayEnd) {
GuardPage = 0;
}
} else {
- Pte[Index] = PageAddress + IA32_PG_RW + IA32_PG_P;
+ Pte[Index] = PageAddress | mAddressEncMask | PAGE_ATTRIBUTE_BITS;
}
PageAddress+= EFI_PAGE_SIZE;
}
@@ -827,77 +996,494 @@ Gen4GPageTable (
}
}
+ if ((PcdGet8 (PcdNullPointerDetectionPropertyMask) & BIT1) != 0) {
+ Pte = (UINT64*)(UINTN)(Pdpte[0] & ~mAddressEncMask & ~(EFI_PAGE_SIZE - 1));
+ if ((Pte[0] & IA32_PG_PS) == 0) {
+ // 4K-page entries are already mapped. Just hide the first one anyway.
+ Pte = (UINT64*)(UINTN)(Pte[0] & ~mAddressEncMask & ~(EFI_PAGE_SIZE - 1));
+ Pte[0] &= ~(UINT64)IA32_PG_P; // Hide page 0
+ } else {
+ // Create 4K-page entries
+ Pages = (UINTN)AllocatePageTableMemory (1);
+ ASSERT (Pages != 0);
+
+ Pte[0] = (UINT64)(Pages | mAddressEncMask | PAGE_ATTRIBUTE_BITS);
+
+ Pte = (UINT64*)Pages;
+ PageAddress = 0;
+ Pte[0] = PageAddress | mAddressEncMask; // Hide page 0 but present left
+ for (Index = 1; Index < EFI_PAGE_SIZE / sizeof (*Pte); Index++) {
+ PageAddress += EFI_PAGE_SIZE;
+ Pte[Index] = PageAddress | mAddressEncMask | PAGE_ATTRIBUTE_BITS;
+ }
+ }
+ }
+
return (UINT32)(UINTN)PageTable;
}
/**
- Set memory cache ability.
+ Checks whether the input token is the current used token.
- @param PageTable PageTable Address
- @param Address Memory Address to change cache ability
- @param Cacheability Cache ability to set
+ @param[in] Token This parameter describes the token that was passed into DispatchProcedure or
+ BroadcastProcedure.
+
+ @retval TRUE The input token is the current used token.
+ @retval FALSE The input token is not the current used token.
+**/
+BOOLEAN
+IsTokenInUse (
+ IN SPIN_LOCK *Token
+ )
+{
+ LIST_ENTRY *Link;
+ PROCEDURE_TOKEN *ProcToken;
+
+ if (Token == NULL) {
+ return FALSE;
+ }
+
+ Link = GetFirstNode (&gSmmCpuPrivate->TokenList);
+ while (!IsNull (&gSmmCpuPrivate->TokenList, Link)) {
+ ProcToken = PROCEDURE_TOKEN_FROM_LINK (Link);
+
+ if (ProcToken->Used && ProcToken->SpinLock == Token) {
+ return TRUE;
+ }
+
+ Link = GetNextNode (&gSmmCpuPrivate->TokenList, Link);
+ }
+
+ return FALSE;
+}
+
+/**
+ Allocate buffer for the SPIN_LOCK and PROCEDURE_TOKEN.
**/
VOID
-SetCacheability (
- IN UINT64 *PageTable,
- IN UINTN Address,
- IN UINT8 Cacheability
+AllocateTokenBuffer (
+ VOID
)
{
- UINTN PTIndex;
- VOID *NewPageTableAddress;
- UINT64 *NewPageTable;
- UINTN Index;
+ UINTN SpinLockSize;
+ UINT32 TokenCountPerChunk;
+ UINTN ProcTokenSize;
+ UINTN Index;
+ PROCEDURE_TOKEN *ProcToken;
+ SPIN_LOCK *SpinLock;
+ UINT8 *SpinLockBuffer;
+ UINT8 *ProcTokenBuffer;
+
+ SpinLockSize = GetSpinLockProperties ();
+ ProcTokenSize = sizeof (PROCEDURE_TOKEN);
+
+ TokenCountPerChunk = FixedPcdGet32 (PcdCpuSmmMpTokenCountPerChunk);
+ ASSERT (TokenCountPerChunk != 0);
+ if (TokenCountPerChunk == 0) {
+ DEBUG ((DEBUG_ERROR, "PcdCpuSmmMpTokenCountPerChunk should not be Zero!\n"));
+ CpuDeadLoop ();
+ }
+ DEBUG ((DEBUG_INFO, "CpuSmm: SpinLock Size = 0x%x, PcdCpuSmmMpTokenCountPerChunk = 0x%x\n", SpinLockSize, TokenCountPerChunk));
+
+ //
+ // Separate the Spin_lock and Proc_token because the alignment requires by Spin_Lock.
+ //
+ SpinLockBuffer = AllocatePool (SpinLockSize * TokenCountPerChunk);
+ ASSERT (SpinLockBuffer != NULL);
+
+ ProcTokenBuffer = AllocatePool (ProcTokenSize * TokenCountPerChunk);
+ ASSERT (ProcTokenBuffer != NULL);
+
+ for (Index = 0; Index < TokenCountPerChunk; Index++) {
+ SpinLock = (SPIN_LOCK *)(SpinLockBuffer + SpinLockSize * Index);
+ InitializeSpinLock (SpinLock);
+
+ ProcToken = (PROCEDURE_TOKEN *)(ProcTokenBuffer + ProcTokenSize * Index);
+ ProcToken->Signature = PROCEDURE_TOKEN_SIGNATURE;
+ ProcToken->SpinLock = SpinLock;
+ ProcToken->Used = FALSE;
+ ProcToken->RunningApCount = 0;
+
+ InsertTailList (&gSmmCpuPrivate->TokenList, &ProcToken->Link);
+ }
+}
+
+/**
+ Find first free token in the allocated token list.
+
+ @retval return the first free PROCEDURE_TOKEN.
+
+**/
+PROCEDURE_TOKEN *
+FindFirstFreeToken (
+ VOID
+ )
+{
+ LIST_ENTRY *Link;
+ PROCEDURE_TOKEN *ProcToken;
- ASSERT ((Address & EFI_PAGE_MASK) == 0);
+ Link = GetFirstNode (&gSmmCpuPrivate->TokenList);
+ while (!IsNull (&gSmmCpuPrivate->TokenList, Link)) {
+ ProcToken = PROCEDURE_TOKEN_FROM_LINK (Link);
- if (sizeof (UINTN) == sizeof (UINT64)) {
- PTIndex = (UINTN)RShiftU64 (Address, 39) & 0x1ff;
- ASSERT (PageTable[PTIndex] & IA32_PG_P);
- PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & gPhyMask);
+ if (!ProcToken->Used) {
+ return ProcToken;
+ }
+
+ Link = GetNextNode (&gSmmCpuPrivate->TokenList, Link);
}
- PTIndex = (UINTN)RShiftU64 (Address, 30) & 0x1ff;
- ASSERT (PageTable[PTIndex] & IA32_PG_P);
- PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & gPhyMask);
+ return NULL;
+}
+
+/**
+ Get the free token.
+
+ If no free token, allocate new tokens then return the free one.
+
+ @retval return the first free PROCEDURE_TOKEN.
+
+**/
+PROCEDURE_TOKEN *
+GetFreeToken (
+ IN UINT32 RunningApsCount
+ )
+{
+ PROCEDURE_TOKEN *NewToken;
+
+ NewToken = FindFirstFreeToken ();
+ if (NewToken == NULL) {
+ AllocateTokenBuffer ();
+ NewToken = FindFirstFreeToken ();
+ }
+ ASSERT (NewToken != NULL);
+
+ NewToken->Used = TRUE;
+ NewToken->RunningApCount = RunningApsCount;
+ AcquireSpinLock (NewToken->SpinLock);
+
+ return NewToken;
+}
+
+/**
+ Checks status of specified AP.
+
+ This function checks whether the specified AP has finished the task assigned
+ by StartupThisAP(), and whether timeout expires.
+
+ @param[in] Token This parameter describes the token that was passed into DispatchProcedure or
+ BroadcastProcedure.
+
+ @retval EFI_SUCCESS Specified AP has finished task assigned by StartupThisAPs().
+ @retval EFI_NOT_READY Specified AP has not finished task and timeout has not expired.
+**/
+EFI_STATUS
+IsApReady (
+ IN SPIN_LOCK *Token
+ )
+{
+ if (AcquireSpinLockOrFail (Token)) {
+ ReleaseSpinLock (Token);
+ return EFI_SUCCESS;
+ }
+
+ return EFI_NOT_READY;
+}
+
+/**
+ Schedule a procedure to run on the specified CPU.
+
+ @param[in] Procedure The address of the procedure to run
+ @param[in] CpuIndex Target CPU Index
+ @param[in,out] ProcArguments The parameter to pass to the procedure
+ @param[in] Token This is an optional parameter that allows the caller to execute the
+ procedure in a blocking or non-blocking fashion. If it is NULL the
+ call is blocking, and the call will not return until the AP has
+ completed the procedure. If the token is not NULL, the call will
+ return immediately. The caller can check whether the procedure has
+ completed with CheckOnProcedure or WaitForProcedure.
+ @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for the APs to finish
+ execution of Procedure, either for blocking or non-blocking mode.
+ Zero means infinity. If the timeout expires before all APs return
+ from Procedure, then Procedure on the failed APs is terminated. If
+ the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
+ If the timeout expires in non-blocking mode, the timeout determined
+ can be through CheckOnProcedure or WaitForProcedure.
+ Note that timeout support is optional. Whether an implementation
+ supports this feature can be determined via the Attributes data
+ member.
+ @param[in,out] CpuStatus This optional pointer may be used to get the status code returned
+ by Procedure when it completes execution on the target AP, or with
+ EFI_TIMEOUT if the Procedure fails to complete within the optional
+ timeout. The implementation will update this variable with
+ EFI_NOT_READY prior to starting Procedure on the target AP.
+
+ @retval EFI_INVALID_PARAMETER CpuNumber not valid
+ @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
+ @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
+ @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
+ @retval EFI_SUCCESS The procedure has been successfully scheduled
+
+**/
+EFI_STATUS
+InternalSmmStartupThisAp (
+ IN EFI_AP_PROCEDURE2 Procedure,
+ IN UINTN CpuIndex,
+ IN OUT VOID *ProcArguments OPTIONAL,
+ IN MM_COMPLETION *Token,
+ IN UINTN TimeoutInMicroseconds,
+ IN OUT EFI_STATUS *CpuStatus
+ )
+{
+ PROCEDURE_TOKEN *ProcToken;
+
+ if (CpuIndex >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus) {
+ DEBUG((DEBUG_ERROR, "CpuIndex(%d) >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus(%d)\n", CpuIndex, gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus));
+ return EFI_INVALID_PARAMETER;
+ }
+ if (CpuIndex == gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu) {
+ DEBUG((DEBUG_ERROR, "CpuIndex(%d) == gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu\n", CpuIndex));
+ return EFI_INVALID_PARAMETER;
+ }
+ if (gSmmCpuPrivate->ProcessorInfo[CpuIndex].ProcessorId == INVALID_APIC_ID) {
+ return EFI_INVALID_PARAMETER;
+ }
+ if (!(*(mSmmMpSyncData->CpuData[CpuIndex].Present))) {
+ if (mSmmMpSyncData->EffectiveSyncMode == SmmCpuSyncModeTradition) {
+ DEBUG((DEBUG_ERROR, "!mSmmMpSyncData->CpuData[%d].Present\n", CpuIndex));
+ }
+ return EFI_INVALID_PARAMETER;
+ }
+ if (gSmmCpuPrivate->Operation[CpuIndex] == SmmCpuRemove) {
+ if (!FeaturePcdGet (PcdCpuHotPlugSupport)) {
+ DEBUG((DEBUG_ERROR, "gSmmCpuPrivate->Operation[%d] == SmmCpuRemove\n", CpuIndex));
+ }
+ return EFI_INVALID_PARAMETER;
+ }
+ if ((TimeoutInMicroseconds != 0) && ((mSmmMp.Attributes & EFI_MM_MP_TIMEOUT_SUPPORTED) == 0)) {
+ return EFI_INVALID_PARAMETER;
+ }
+ if (Procedure == NULL) {
+ return EFI_INVALID_PARAMETER;
+ }
+
+ AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
+
+ mSmmMpSyncData->CpuData[CpuIndex].Procedure = Procedure;
+ mSmmMpSyncData->CpuData[CpuIndex].Parameter = ProcArguments;
+ if (Token != NULL) {
+ ProcToken= GetFreeToken (1);
+ mSmmMpSyncData->CpuData[CpuIndex].Token = ProcToken;
+ *Token = (MM_COMPLETION)ProcToken->SpinLock;
+ }
+ mSmmMpSyncData->CpuData[CpuIndex].Status = CpuStatus;
+ if (mSmmMpSyncData->CpuData[CpuIndex].Status != NULL) {
+ *mSmmMpSyncData->CpuData[CpuIndex].Status = EFI_NOT_READY;
+ }
+
+ ReleaseSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);
+
+ if (Token == NULL) {
+ AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
+ ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
+ }
+
+ return EFI_SUCCESS;
+}
+
+/**
+ Worker function to execute a caller provided function on all enabled APs.
+
+ @param[in] Procedure A pointer to the function to be run on
+ enabled APs of the system.
+ @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
+ APs to return from Procedure, either for
+ blocking or non-blocking mode.
+ @param[in,out] ProcedureArguments The parameter passed into Procedure for
+ all APs.
+ @param[in,out] Token This is an optional parameter that allows the caller to execute the
+ procedure in a blocking or non-blocking fashion. If it is NULL the
+ call is blocking, and the call will not return until the AP has
+ completed the procedure. If the token is not NULL, the call will
+ return immediately. The caller can check whether the procedure has
+ completed with CheckOnProcedure or WaitForProcedure.
+ @param[in,out] CPUStatus This optional pointer may be used to get the status code returned
+ by Procedure when it completes execution on the target AP, or with
+ EFI_TIMEOUT if the Procedure fails to complete within the optional
+ timeout. The implementation will update this variable with
+ EFI_NOT_READY prior to starting Procedure on the target AP.
+
+
+ @retval EFI_SUCCESS In blocking mode, all APs have finished before
+ the timeout expired.
+ @retval EFI_SUCCESS In non-blocking mode, function has been dispatched
+ to all enabled APs.
+ @retval others Failed to Startup all APs.
+
+**/
+EFI_STATUS
+InternalSmmStartupAllAPs (
+ IN EFI_AP_PROCEDURE2 Procedure,
+ IN UINTN TimeoutInMicroseconds,
+ IN OUT VOID *ProcedureArguments OPTIONAL,
+ IN OUT MM_COMPLETION *Token,
+ IN OUT EFI_STATUS *CPUStatus
+ )
+{
+ UINTN Index;
+ UINTN CpuCount;
+ PROCEDURE_TOKEN *ProcToken;
+
+ if ((TimeoutInMicroseconds != 0) && ((mSmmMp.Attributes & EFI_MM_MP_TIMEOUT_SUPPORTED) == 0)) {
+ return EFI_INVALID_PARAMETER;
+ }
+ if (Procedure == NULL) {
+ return EFI_INVALID_PARAMETER;
+ }
+
+ CpuCount = 0;
+ for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
+ if (IsPresentAp (Index)) {
+ CpuCount ++;
+
+ if (gSmmCpuPrivate->Operation[Index] == SmmCpuRemove) {
+ return EFI_INVALID_PARAMETER;
+ }
+
+ if (!AcquireSpinLockOrFail(mSmmMpSyncData->CpuData[Index].Busy)) {
+ return EFI_NOT_READY;
+ }
+ ReleaseSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
+ }
+ }
+ if (CpuCount == 0) {
+ return EFI_NOT_STARTED;
+ }
+
+ if (Token != NULL) {
+ ProcToken = GetFreeToken ((UINT32)mMaxNumberOfCpus);
+ *Token = (MM_COMPLETION)ProcToken->SpinLock;
+ } else {
+ ProcToken = NULL;
+ }
//
- // A perfect implementation should check the original cacheability with the
- // one being set, and break a 2M page entry into pieces only when they
- // disagreed.
+ // Make sure all BUSY should be acquired.
//
- PTIndex = (UINTN)RShiftU64 (Address, 21) & 0x1ff;
- if ((PageTable[PTIndex] & IA32_PG_PS) != 0) {
- //
- // Allocate a page from SMRAM
- //
- NewPageTableAddress = AllocatePages (1);
- ASSERT (NewPageTableAddress != NULL);
+ // Because former code already check mSmmMpSyncData->CpuData[***].Busy for each AP.
+ // Here code always use AcquireSpinLock instead of AcquireSpinLockOrFail for not
+ // block mode.
+ //
+ for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
+ if (IsPresentAp (Index)) {
+ AcquireSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
+ }
+ }
- NewPageTable = (UINT64 *)NewPageTableAddress;
+ for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
+ if (IsPresentAp (Index)) {
+ mSmmMpSyncData->CpuData[Index].Procedure = (EFI_AP_PROCEDURE2) Procedure;
+ mSmmMpSyncData->CpuData[Index].Parameter = ProcedureArguments;
+ if (ProcToken != NULL) {
+ mSmmMpSyncData->CpuData[Index].Token = ProcToken;
+ }
+ if (CPUStatus != NULL) {
+ mSmmMpSyncData->CpuData[Index].Status = &CPUStatus[Index];
+ if (mSmmMpSyncData->CpuData[Index].Status != NULL) {
+ *mSmmMpSyncData->CpuData[Index].Status = EFI_NOT_READY;
+ }
+ }
+ } else {
+ //
+ // PI spec requirement:
+ // For every excluded processor, the array entry must contain a value of EFI_NOT_STARTED.
+ //
+ if (CPUStatus != NULL) {
+ CPUStatus[Index] = EFI_NOT_STARTED;
+ }
- for (Index = 0; Index < 0x200; Index++) {
- NewPageTable[Index] = PageTable[PTIndex];
- if ((NewPageTable[Index] & IA32_PG_PAT_2M) != 0) {
- NewPageTable[Index] &= ~((UINT64)IA32_PG_PAT_2M);
- NewPageTable[Index] |= (UINT64)IA32_PG_PAT_4K;
+ //
+ // Decrease the count to mark this processor(AP or BSP) as finished.
+ //
+ if (ProcToken != NULL) {
+ WaitForSemaphore (&ProcToken->RunningApCount);
}
- NewPageTable[Index] |= (UINT64)(Index << EFI_PAGE_SHIFT);
}
+ }
+
+ ReleaseAllAPs ();
- PageTable[PTIndex] = ((UINTN)NewPageTableAddress & gPhyMask) | IA32_PG_P;
+ if (Token == NULL) {
+ //
+ // Make sure all APs have completed their tasks.
+ //
+ WaitForAllAPsNotBusy (TRUE);
}
- ASSERT (PageTable[PTIndex] & IA32_PG_P);
- PageTable = (UINT64*)(UINTN)(PageTable[PTIndex] & gPhyMask);
+ return EFI_SUCCESS;
+}
+
+/**
+ ISO C99 6.5.2.2 "Function calls", paragraph 9:
+ If the function is defined with a type that is not compatible with
+ the type (of the expression) pointed to by the expression that
+ denotes the called function, the behavior is undefined.
+
+ So add below wrapper function to convert between EFI_AP_PROCEDURE
+ and EFI_AP_PROCEDURE2.
+
+ Wrapper for Procedures.
+
+ @param[in] Buffer Pointer to PROCEDURE_WRAPPER buffer.
- PTIndex = (UINTN)RShiftU64 (Address, 12) & 0x1ff;
- ASSERT (PageTable[PTIndex] & IA32_PG_P);
- PageTable[PTIndex] &= ~((UINT64)((IA32_PG_PAT_4K | IA32_PG_CD | IA32_PG_WT)));
- PageTable[PTIndex] |= (UINT64)Cacheability;
+**/
+EFI_STATUS
+EFIAPI
+ProcedureWrapper (
+ IN VOID *Buffer
+ )
+{
+ PROCEDURE_WRAPPER *Wrapper;
+
+ Wrapper = Buffer;
+ Wrapper->Procedure (Wrapper->ProcedureArgument);
+
+ return EFI_SUCCESS;
}
+/**
+ Schedule a procedure to run on the specified CPU in blocking mode.
+
+ @param[in] Procedure The address of the procedure to run
+ @param[in] CpuIndex Target CPU Index
+ @param[in, out] ProcArguments The parameter to pass to the procedure
+
+ @retval EFI_INVALID_PARAMETER CpuNumber not valid
+ @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
+ @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
+ @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
+ @retval EFI_SUCCESS The procedure has been successfully scheduled
+
+**/
+EFI_STATUS
+EFIAPI
+SmmBlockingStartupThisAp (
+ IN EFI_AP_PROCEDURE Procedure,
+ IN UINTN CpuIndex,
+ IN OUT VOID *ProcArguments OPTIONAL
+ )
+{
+ PROCEDURE_WRAPPER Wrapper;
+
+ Wrapper.Procedure = Procedure;
+ Wrapper.ProcedureArgument = ProcArguments;
+
+ //
+ // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
+ //
+ return InternalSmmStartupThisAp (ProcedureWrapper, CpuIndex, &Wrapper, NULL, 0, NULL);
+}
/**
Schedule a procedure to run on the specified CPU.
@@ -921,23 +1507,83 @@ SmmStartupThisAp (
IN OUT VOID *ProcArguments OPTIONAL
)
{
- if (CpuIndex >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus ||
- CpuIndex == gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu ||
- !mSmmMpSyncData->CpuData[CpuIndex].Present ||
- gSmmCpuPrivate->Operation[CpuIndex] == SmmCpuRemove ||
- !AcquireSpinLockOrFail (&mSmmMpSyncData->CpuData[CpuIndex].Busy)) {
- return EFI_INVALID_PARAMETER;
+ MM_COMPLETION Token;
+
+ gSmmCpuPrivate->ApWrapperFunc[CpuIndex].Procedure = Procedure;
+ gSmmCpuPrivate->ApWrapperFunc[CpuIndex].ProcedureArgument = ProcArguments;
+
+ //
+ // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
+ //
+ return InternalSmmStartupThisAp (
+ ProcedureWrapper,
+ CpuIndex,
+ &gSmmCpuPrivate->ApWrapperFunc[CpuIndex],
+ FeaturePcdGet (PcdCpuSmmBlockStartupThisAp) ? NULL : &Token,
+ 0,
+ NULL
+ );
+}
+
+/**
+ This function sets DR6 & DR7 according to SMM save state, before running SMM C code.
+ They are useful when you want to enable hardware breakpoints in SMM without entry SMM mode.
+
+ NOTE: It might not be appreciated in runtime since it might
+ conflict with OS debugging facilities. Turn them off in RELEASE.
+
+ @param CpuIndex CPU Index
+
+**/
+VOID
+EFIAPI
+CpuSmmDebugEntry (
+ IN UINTN CpuIndex
+ )
+{
+ SMRAM_SAVE_STATE_MAP *CpuSaveState;
+
+ if (FeaturePcdGet (PcdCpuSmmDebug)) {
+ ASSERT(CpuIndex < mMaxNumberOfCpus);
+ CpuSaveState = (SMRAM_SAVE_STATE_MAP *)gSmmCpuPrivate->CpuSaveState[CpuIndex];
+ if (mSmmSaveStateRegisterLma == EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT) {
+ AsmWriteDr6 (CpuSaveState->x86._DR6);
+ AsmWriteDr7 (CpuSaveState->x86._DR7);
+ } else {
+ AsmWriteDr6 ((UINTN)CpuSaveState->x64._DR6);
+ AsmWriteDr7 ((UINTN)CpuSaveState->x64._DR7);
+ }
}
+}
- mSmmMpSyncData->CpuData[CpuIndex].Procedure = Procedure;
- mSmmMpSyncData->CpuData[CpuIndex].Parameter = ProcArguments;
- ReleaseSemaphore (&mSmmMpSyncData->CpuData[CpuIndex].Run);
+/**
+ This function restores DR6 & DR7 to SMM save state.
+
+ NOTE: It might not be appreciated in runtime since it might
+ conflict with OS debugging facilities. Turn them off in RELEASE.
+
+ @param CpuIndex CPU Index
- if (FeaturePcdGet (PcdCpuSmmBlockStartupThisAp)) {
- AcquireSpinLock (&mSmmMpSyncData->CpuData[CpuIndex].Busy);
- ReleaseSpinLock (&mSmmMpSyncData->CpuData[CpuIndex].Busy);
+**/
+VOID
+EFIAPI
+CpuSmmDebugExit (
+ IN UINTN CpuIndex
+ )
+{
+ SMRAM_SAVE_STATE_MAP *CpuSaveState;
+
+ if (FeaturePcdGet (PcdCpuSmmDebug)) {
+ ASSERT(CpuIndex < mMaxNumberOfCpus);
+ CpuSaveState = (SMRAM_SAVE_STATE_MAP *)gSmmCpuPrivate->CpuSaveState[CpuIndex];
+ if (mSmmSaveStateRegisterLma == EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT) {
+ CpuSaveState->x86._DR7 = (UINT32)AsmReadDr7 ();
+ CpuSaveState->x86._DR6 = (UINT32)AsmReadDr6 ();
+ } else {
+ CpuSaveState->x64._DR7 = AsmReadDr7 ();
+ CpuSaveState->x64._DR6 = AsmReadDr6 ();
+ }
}
- return EFI_SUCCESS;
}
/**
@@ -952,17 +1598,28 @@ SmiRendezvous (
IN UINTN CpuIndex
)
{
- EFI_STATUS Status;
- BOOLEAN ValidSmi;
- BOOLEAN IsBsp;
- BOOLEAN BspInProgress;
- UINTN Index;
- UINTN Cr2;
+ EFI_STATUS Status;
+ BOOLEAN ValidSmi;
+ BOOLEAN IsBsp;
+ BOOLEAN BspInProgress;
+ UINTN Index;
+ UINTN Cr2;
+
+ ASSERT(CpuIndex < mMaxNumberOfCpus);
//
- // Save Cr2 because Page Fault exception in SMM may override its value
+ // Save Cr2 because Page Fault exception in SMM may override its value,
+ // when using on-demand paging for above 4G memory.
//
- Cr2 = AsmReadCr2 ();
+ Cr2 = 0;
+ SaveCr2 (&Cr2);
+
+ //
+ // Call the user register Startup function first.
+ //
+ if (mSmmMpSyncData->StartupProcedure != NULL) {
+ mSmmMpSyncData->StartupProcedure (mSmmMpSyncData->StartupProcArgs);
+ }
//
// Perform CPU specific entry hooks
@@ -978,7 +1635,7 @@ SmiRendezvous (
// Determine if BSP has been already in progress. Note this must be checked after
// ValidSmi because BSP may clear a valid SMI source after checking in.
//
- BspInProgress = mSmmMpSyncData->InsideSmm;
+ BspInProgress = *mSmmMpSyncData->InsideSmm;
if (!BspInProgress && !ValidSmi) {
//
@@ -993,7 +1650,7 @@ SmiRendezvous (
//
// Signal presence of this processor
//
- if (ReleaseSemaphore (&mSmmMpSyncData->Counter) == 0) {
+ if (ReleaseSemaphore (mSmmMpSyncData->Counter) == 0) {
//
// BSP has already ended the synchronization, so QUIT!!!
//
@@ -1001,7 +1658,7 @@ SmiRendezvous (
//
// Wait for BSP's signal to finish SMI
//
- while (mSmmMpSyncData->AllCpusInSync) {
+ while (*mSmmMpSyncData->AllCpusInSync) {
CpuPause ();
}
goto Exit;
@@ -1013,14 +1670,7 @@ SmiRendezvous (
// E.g., with Relaxed AP flow, SmmStartupThisAp() may be called immediately
// after AP's present flag is detected.
//
- InitializeSpinLock (&mSmmMpSyncData->CpuData[CpuIndex].Busy);
- }
-
- //
- // Try to enable NX
- //
- if (mXdSupported) {
- ActivateXd ();
+ InitializeSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
}
if (FeaturePcdGet (PcdCpuSmmProfileEnable)) {
@@ -1091,30 +1741,102 @@ SmiRendezvous (
// BSP Handler is always called with a ValidSmi == TRUE
//
BSPHandler (CpuIndex, mSmmMpSyncData->EffectiveSyncMode);
-
} else {
APHandler (CpuIndex, ValidSmi, mSmmMpSyncData->EffectiveSyncMode);
}
}
- ASSERT (mSmmMpSyncData->CpuData[CpuIndex].Run == 0);
+ ASSERT (*mSmmMpSyncData->CpuData[CpuIndex].Run == 0);
//
// Wait for BSP's signal to exit SMI
//
- while (mSmmMpSyncData->AllCpusInSync) {
+ while (*mSmmMpSyncData->AllCpusInSync) {
CpuPause ();
}
}
Exit:
SmmCpuFeaturesRendezvousExit (CpuIndex);
+
//
// Restore Cr2
//
- AsmWriteCr2 (Cr2);
+ RestoreCr2 (Cr2);
+}
+
+/**
+ Allocate buffer for SpinLock and Wrapper function buffer.
+
+**/
+VOID
+InitializeDataForMmMp (
+ VOID
+ )
+{
+ gSmmCpuPrivate->ApWrapperFunc = AllocatePool (sizeof (PROCEDURE_WRAPPER) * gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus);
+ ASSERT (gSmmCpuPrivate->ApWrapperFunc != NULL);
+
+ InitializeListHead (&gSmmCpuPrivate->TokenList);
+
+ AllocateTokenBuffer ();
}
+/**
+ Allocate buffer for all semaphores and spin locks.
+
+**/
+VOID
+InitializeSmmCpuSemaphores (
+ VOID
+ )
+{
+ UINTN ProcessorCount;
+ UINTN TotalSize;
+ UINTN GlobalSemaphoresSize;
+ UINTN CpuSemaphoresSize;
+ UINTN SemaphoreSize;
+ UINTN Pages;
+ UINTN *SemaphoreBlock;
+ UINTN SemaphoreAddr;
+
+ SemaphoreSize = GetSpinLockProperties ();
+ ProcessorCount = gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus;
+ GlobalSemaphoresSize = (sizeof (SMM_CPU_SEMAPHORE_GLOBAL) / sizeof (VOID *)) * SemaphoreSize;
+ CpuSemaphoresSize = (sizeof (SMM_CPU_SEMAPHORE_CPU) / sizeof (VOID *)) * ProcessorCount * SemaphoreSize;
+ TotalSize = GlobalSemaphoresSize + CpuSemaphoresSize;
+ DEBUG((EFI_D_INFO, "One Semaphore Size = 0x%x\n", SemaphoreSize));
+ DEBUG((EFI_D_INFO, "Total Semaphores Size = 0x%x\n", TotalSize));
+ Pages = EFI_SIZE_TO_PAGES (TotalSize);
+ SemaphoreBlock = AllocatePages (Pages);
+ ASSERT (SemaphoreBlock != NULL);
+ ZeroMem (SemaphoreBlock, TotalSize);
+
+ SemaphoreAddr = (UINTN)SemaphoreBlock;
+ mSmmCpuSemaphores.SemaphoreGlobal.Counter = (UINT32 *)SemaphoreAddr;
+ SemaphoreAddr += SemaphoreSize;
+ mSmmCpuSemaphores.SemaphoreGlobal.InsideSmm = (BOOLEAN *)SemaphoreAddr;
+ SemaphoreAddr += SemaphoreSize;
+ mSmmCpuSemaphores.SemaphoreGlobal.AllCpusInSync = (BOOLEAN *)SemaphoreAddr;
+ SemaphoreAddr += SemaphoreSize;
+ mSmmCpuSemaphores.SemaphoreGlobal.PFLock = (SPIN_LOCK *)SemaphoreAddr;
+ SemaphoreAddr += SemaphoreSize;
+ mSmmCpuSemaphores.SemaphoreGlobal.CodeAccessCheckLock
+ = (SPIN_LOCK *)SemaphoreAddr;
+ SemaphoreAddr += SemaphoreSize;
+
+ SemaphoreAddr = (UINTN)SemaphoreBlock + GlobalSemaphoresSize;
+ mSmmCpuSemaphores.SemaphoreCpu.Busy = (SPIN_LOCK *)SemaphoreAddr;
+ SemaphoreAddr += ProcessorCount * SemaphoreSize;
+ mSmmCpuSemaphores.SemaphoreCpu.Run = (UINT32 *)SemaphoreAddr;
+ SemaphoreAddr += ProcessorCount * SemaphoreSize;
+ mSmmCpuSemaphores.SemaphoreCpu.Present = (BOOLEAN *)SemaphoreAddr;
+
+ mPFLock = mSmmCpuSemaphores.SemaphoreGlobal.PFLock;
+ mConfigSmmCodeAccessCheckLock = mSmmCpuSemaphores.SemaphoreGlobal.CodeAccessCheckLock;
+
+ mSemaphoreSize = SemaphoreSize;
+}
/**
Initialize un-cacheable data.
@@ -1126,7 +1848,13 @@ InitializeMpSyncData (
VOID
)
{
+ UINTN CpuIndex;
+
if (mSmmMpSyncData != NULL) {
+ //
+ // mSmmMpSyncDataSize includes one structure of SMM_DISPATCHER_MP_SYNC_DATA, one
+ // CpuData array of SMM_CPU_DATA_BLOCK and one CandidateBsp array of BOOLEAN.
+ //
ZeroMem (mSmmMpSyncData, mSmmMpSyncDataSize);
mSmmMpSyncData->CpuData = (SMM_CPU_DATA_BLOCK *)((UINT8 *)mSmmMpSyncData + sizeof (SMM_DISPATCHER_MP_SYNC_DATA));
mSmmMpSyncData->CandidateBsp = (BOOLEAN *)(mSmmMpSyncData->CpuData + gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus);
@@ -1136,29 +1864,72 @@ InitializeMpSyncData (
//
mSmmMpSyncData->BspIndex = (UINT32)-1;
}
- mSmmMpSyncData->EffectiveSyncMode = (SMM_CPU_SYNC_MODE) PcdGet8 (PcdCpuSmmSyncMode);
+ mSmmMpSyncData->EffectiveSyncMode = mCpuSmmSyncMode;
+
+ mSmmMpSyncData->Counter = mSmmCpuSemaphores.SemaphoreGlobal.Counter;
+ mSmmMpSyncData->InsideSmm = mSmmCpuSemaphores.SemaphoreGlobal.InsideSmm;
+ mSmmMpSyncData->AllCpusInSync = mSmmCpuSemaphores.SemaphoreGlobal.AllCpusInSync;
+ ASSERT (mSmmMpSyncData->Counter != NULL && mSmmMpSyncData->InsideSmm != NULL &&
+ mSmmMpSyncData->AllCpusInSync != NULL);
+ *mSmmMpSyncData->Counter = 0;
+ *mSmmMpSyncData->InsideSmm = FALSE;
+ *mSmmMpSyncData->AllCpusInSync = FALSE;
+
+ for (CpuIndex = 0; CpuIndex < gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus; CpuIndex ++) {
+ mSmmMpSyncData->CpuData[CpuIndex].Busy =
+ (SPIN_LOCK *)((UINTN)mSmmCpuSemaphores.SemaphoreCpu.Busy + mSemaphoreSize * CpuIndex);
+ mSmmMpSyncData->CpuData[CpuIndex].Run =
+ (UINT32 *)((UINTN)mSmmCpuSemaphores.SemaphoreCpu.Run + mSemaphoreSize * CpuIndex);
+ mSmmMpSyncData->CpuData[CpuIndex].Present =
+ (BOOLEAN *)((UINTN)mSmmCpuSemaphores.SemaphoreCpu.Present + mSemaphoreSize * CpuIndex);
+ *(mSmmMpSyncData->CpuData[CpuIndex].Busy) = 0;
+ *(mSmmMpSyncData->CpuData[CpuIndex].Run) = 0;
+ *(mSmmMpSyncData->CpuData[CpuIndex].Present) = FALSE;
+ }
}
}
/**
Initialize global data for MP synchronization.
- @param Stacks Base address of SMI stack buffer for all processors.
- @param StackSize Stack size for each processor in SMM.
+ @param Stacks Base address of SMI stack buffer for all processors.
+ @param StackSize Stack size for each processor in SMM.
+ @param ShadowStackSize Shadow Stack size for each processor in SMM.
**/
UINT32
InitializeMpServiceData (
IN VOID *Stacks,
- IN UINTN StackSize
+ IN UINTN StackSize,
+ IN UINTN ShadowStackSize
)
{
UINT32 Cr3;
UINTN Index;
- MTRR_SETTINGS *Mtrr;
- PROCESSOR_SMM_DESCRIPTOR *Psd;
UINT8 *GdtTssTables;
UINTN GdtTableStepSize;
+ CPUID_VERSION_INFO_EDX RegEdx;
+
+ //
+ // Determine if this CPU supports machine check
+ //
+ AsmCpuid (CPUID_VERSION_INFO, NULL, NULL, NULL, &RegEdx.Uint32);
+ mMachineCheckSupported = (BOOLEAN)(RegEdx.Bits.MCA == 1);
+
+ //
+ // Allocate memory for all locks and semaphores
+ //
+ InitializeSmmCpuSemaphores ();
+
+ //
+ // Initialize mSmmMpSyncData
+ //
+ mSmmMpSyncDataSize = sizeof (SMM_DISPATCHER_MP_SYNC_DATA) +
+ (sizeof (SMM_CPU_DATA_BLOCK) + sizeof (BOOLEAN)) * gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus;
+ mSmmMpSyncData = (SMM_DISPATCHER_MP_SYNC_DATA*) AllocatePages (EFI_SIZE_TO_PAGES (mSmmMpSyncDataSize));
+ ASSERT (mSmmMpSyncData != NULL);
+ mCpuSmmSyncMode = (SMM_CPU_SYNC_MODE)PcdGet8 (PcdCpuSmmSyncMode);
+ InitializeMpSyncData ();
//
// Initialize physical address mask
@@ -1176,45 +1947,27 @@ InitializeMpServiceData (
GdtTssTables = InitGdt (Cr3, &GdtTableStepSize);
//
- // Initialize PROCESSOR_SMM_DESCRIPTOR for each CPU
+ // Install SMI handler for each CPU
//
for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
- Psd = (PROCESSOR_SMM_DESCRIPTOR *)(VOID *)(UINTN)(mCpuHotPlugData.SmBase[Index] + SMM_PSD_OFFSET);
- CopyMem (Psd, &gcPsd, sizeof (gcPsd));
- Psd->SmmGdtPtr = (UINT64)(UINTN)(GdtTssTables + GdtTableStepSize * Index);
- Psd->SmmGdtSize = gcSmiGdtr.Limit + 1;
-
- //
- // Install SMI handler
- //
InstallSmiHandler (
Index,
(UINT32)mCpuHotPlugData.SmBase[Index],
- (VOID*)((UINTN)Stacks + (StackSize * Index)),
+ (VOID*)((UINTN)Stacks + (StackSize + ShadowStackSize) * Index),
StackSize,
- (UINTN)Psd->SmmGdtPtr,
- Psd->SmmGdtSize,
+ (UINTN)(GdtTssTables + GdtTableStepSize * Index),
+ gcSmiGdtr.Limit + 1,
gcSmiIdtr.Base,
gcSmiIdtr.Limit + 1,
Cr3
);
}
- //
- // Initialize mSmmMpSyncData
- //
- mSmmMpSyncDataSize = sizeof (SMM_DISPATCHER_MP_SYNC_DATA) +
- (sizeof (SMM_CPU_DATA_BLOCK) + sizeof (BOOLEAN)) * gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus;
- mSmmMpSyncData = (SMM_DISPATCHER_MP_SYNC_DATA*) AllocatePages (EFI_SIZE_TO_PAGES (mSmmMpSyncDataSize));
- ASSERT (mSmmMpSyncData != NULL);
- InitializeMpSyncData ();
-
//
// Record current MTRR settings
//
- ZeroMem(gSmiMtrrs, sizeof (gSmiMtrrs));
- Mtrr = (MTRR_SETTINGS*)gSmiMtrrs;
- MtrrGetAllMtrrs (Mtrr);
+ ZeroMem (&gSmiMtrrs, sizeof (gSmiMtrrs));
+ MtrrGetAllMtrrs (&gSmiMtrrs);
return Cr3;
}
@@ -1242,3 +1995,40 @@ RegisterSmmEntry (
gSmmCpuPrivate->SmmCoreEntry = SmmEntryPoint;
return EFI_SUCCESS;
}
+
+/**
+
+ Register the SMM Foundation entry point.
+
+ @param[in] Procedure A pointer to the code stream to be run on the designated target AP
+ of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
+ with the related definitions of
+ EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
+ If caller may pass a value of NULL to deregister any existing
+ startup procedure.
+ @param[in,out] ProcedureArguments Allows the caller to pass a list of parameters to the code that is
+ run by the AP. It is an optional common mailbox between APs and
+ the caller to share information
+
+ @retval EFI_SUCCESS The Procedure has been set successfully.
+ @retval EFI_INVALID_PARAMETER The Procedure is NULL but ProcedureArguments not NULL.
+
+**/
+EFI_STATUS
+RegisterStartupProcedure (
+ IN EFI_AP_PROCEDURE Procedure,
+ IN OUT VOID *ProcedureArguments OPTIONAL
+ )
+{
+ if (Procedure == NULL && ProcedureArguments != NULL) {
+ return EFI_INVALID_PARAMETER;
+ }
+ if (mSmmMpSyncData == NULL) {
+ return EFI_NOT_READY;
+ }
+
+ mSmmMpSyncData->StartupProcedure = Procedure;
+ mSmmMpSyncData->StartupProcArgs = ProcedureArguments;
+
+ return EFI_SUCCESS;
+}