X-Git-Url: https://git.proxmox.com/?p=mirror_edk2.git;a=blobdiff_plain;f=UefiCpuPkg%2FLibrary%2FMpInitLib%2FMpLib.c;h=33374888922a85d80fbe4f4735f1f24446b03926;hp=0495b0f8a4bda8a27273cb7eb078a0c33fbdfd1d;hb=0acd869796ded1266e69487dff717cd69d6031f9;hpb=367284e7735478b0c4ad37d8f6bf17d3c4a473d0
diff --git a/UefiCpuPkg/Library/MpInitLib/MpLib.c b/UefiCpuPkg/Library/MpInitLib/MpLib.c
index 0495b0f8a4..3337488892 100644
--- a/UefiCpuPkg/Library/MpInitLib/MpLib.c
+++ b/UefiCpuPkg/Library/MpInitLib/MpLib.c
@@ -1,14 +1,8 @@
/** @file
CPU MP Initialize Library common functions.
- Copyright (c) 2016, Intel Corporation. All rights reserved.
- This program and the accompanying materials
- are licensed and made available under the terms and conditions of the BSD License
- which accompanies this distribution. The full text of the license may be found at
- http://opensource.org/licenses/bsd-license.php
-
- THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
- WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+ Copyright (c) 2016 - 2018, Intel Corporation. All rights reserved.
+ SPDX-License-Identifier: BSD-2-Clause-Patent
**/
@@ -18,8 +12,11 @@ EFI_GUID mCpuInitMpLibHobGuid = CPU_INIT_MP_LIB_HOB_GUID;
/**
The function will check if BSP Execute Disable is enabled.
- DxeIpl may have enabled Execute Disable for BSP,
- APs need to get the status and sync up the settings.
+
+ DxeIpl may have enabled Execute Disable for BSP, APs need to
+ get the status and sync up the settings.
+ If BSP's CR0.Paging is not set, BSP execute Disble feature is
+ not working actually.
@retval TRUE BSP Execute Disable is enabled.
@retval FALSE BSP Execute Disable is not enabled.
@@ -33,23 +30,30 @@ IsBspExecuteDisableEnabled (
CPUID_EXTENDED_CPU_SIG_EDX Edx;
MSR_IA32_EFER_REGISTER EferMsr;
BOOLEAN Enabled;
+ IA32_CR0 Cr0;
Enabled = FALSE;
- AsmCpuid (CPUID_EXTENDED_FUNCTION, &Eax, NULL, NULL, NULL);
- if (Eax >= CPUID_EXTENDED_CPU_SIG) {
- AsmCpuid (CPUID_EXTENDED_CPU_SIG, NULL, NULL, NULL, &Edx.Uint32);
+ Cr0.UintN = AsmReadCr0 ();
+ if (Cr0.Bits.PG != 0) {
//
- // CPUID 0x80000001
- // Bit 20: Execute Disable Bit available.
+ // If CR0 Paging bit is set
//
- if (Edx.Bits.NX != 0) {
- EferMsr.Uint64 = AsmReadMsr64 (MSR_IA32_EFER);
+ AsmCpuid (CPUID_EXTENDED_FUNCTION, &Eax, NULL, NULL, NULL);
+ if (Eax >= CPUID_EXTENDED_CPU_SIG) {
+ AsmCpuid (CPUID_EXTENDED_CPU_SIG, NULL, NULL, NULL, &Edx.Uint32);
//
- // MSR 0xC0000080
- // Bit 11: Execute Disable Bit enable.
+ // CPUID 0x80000001
+ // Bit 20: Execute Disable Bit available.
//
- if (EferMsr.Bits.NXE != 0) {
- Enabled = TRUE;
+ if (Edx.Bits.NX != 0) {
+ EferMsr.Uint64 = AsmReadMsr64 (MSR_IA32_EFER);
+ //
+ // MSR 0xC0000080
+ // Bit 11: Execute Disable Bit enable.
+ //
+ if (EferMsr.Bits.NXE != 0) {
+ Enabled = TRUE;
+ }
}
}
}
@@ -109,6 +113,53 @@ SetApState (
ReleaseSpinLock (&CpuData->ApLock);
}
+/**
+ Save BSP's local APIC timer setting.
+
+ @param[in] CpuMpData Pointer to CPU MP Data
+**/
+VOID
+SaveLocalApicTimerSetting (
+ IN CPU_MP_DATA *CpuMpData
+ )
+{
+ //
+ // Record the current local APIC timer setting of BSP
+ //
+ GetApicTimerState (
+ &CpuMpData->DivideValue,
+ &CpuMpData->PeriodicMode,
+ &CpuMpData->Vector
+ );
+ CpuMpData->CurrentTimerCount = GetApicTimerCurrentCount ();
+ CpuMpData->TimerInterruptState = GetApicTimerInterruptState ();
+}
+
+/**
+ Sync local APIC timer setting from BSP to AP.
+
+ @param[in] CpuMpData Pointer to CPU MP Data
+**/
+VOID
+SyncLocalApicTimerSetting (
+ IN CPU_MP_DATA *CpuMpData
+ )
+{
+ //
+ // Sync local APIC timer setting from BSP to AP
+ //
+ InitializeApicTimer (
+ CpuMpData->DivideValue,
+ CpuMpData->CurrentTimerCount,
+ CpuMpData->PeriodicMode,
+ CpuMpData->Vector
+ );
+ //
+ // Disable AP's local APIC timer interrupt
+ //
+ DisableApicTimerInterrupt ();
+}
+
/**
Save the volatile registers required to be restored following INIT IPI.
@@ -138,6 +189,10 @@ SaveVolatileRegisters (
VolatileRegisters->Dr6 = AsmReadDr6 ();
VolatileRegisters->Dr7 = AsmReadDr7 ();
}
+
+ AsmReadGdtr (&VolatileRegisters->Gdtr);
+ AsmReadIdtr (&VolatileRegisters->Idtr);
+ VolatileRegisters->Tr = AsmReadTr ();
}
/**
@@ -154,10 +209,11 @@ RestoreVolatileRegisters (
)
{
CPUID_VERSION_INFO_EDX VersionInfoEdx;
+ IA32_TSS_DESCRIPTOR *Tss;
- AsmWriteCr0 (VolatileRegisters->Cr0);
AsmWriteCr3 (VolatileRegisters->Cr3);
AsmWriteCr4 (VolatileRegisters->Cr4);
+ AsmWriteCr0 (VolatileRegisters->Cr0);
if (IsRestoreDr) {
AsmCpuid (CPUID_VERSION_INFO, NULL, NULL, NULL, &VersionInfoEdx.Uint32);
@@ -174,6 +230,18 @@ RestoreVolatileRegisters (
AsmWriteDr7 (VolatileRegisters->Dr7);
}
}
+
+ AsmWriteGdtr (&VolatileRegisters->Gdtr);
+ AsmWriteIdtr (&VolatileRegisters->Idtr);
+ if (VolatileRegisters->Tr != 0 &&
+ VolatileRegisters->Tr < VolatileRegisters->Gdtr.Limit) {
+ Tss = (IA32_TSS_DESCRIPTOR *)(VolatileRegisters->Gdtr.Base +
+ VolatileRegisters->Tr);
+ if (Tss->Bits.P == 1) {
+ Tss->Bits.Type &= 0xD; // 1101 - Clear busy bit just in case
+ AsmWriteTr (VolatileRegisters->Tr);
+ }
+ }
}
/**
@@ -256,6 +324,7 @@ SortApicId (
CPU_INFO_IN_HOB CpuInfo;
UINT32 ApCount;
CPU_INFO_IN_HOB *CpuInfoInHob;
+ volatile UINT32 *StartupApSignal;
ApCount = CpuMpData->CpuCount - 1;
CpuInfoInHob = (CPU_INFO_IN_HOB *) (UINTN) CpuMpData->CpuInfoInHob;
@@ -280,6 +349,14 @@ SortApicId (
sizeof (CPU_INFO_IN_HOB)
);
CopyMem (&CpuInfoInHob[Index1], &CpuInfo, sizeof (CPU_INFO_IN_HOB));
+
+ //
+ // Also exchange the StartupApSignal.
+ //
+ StartupApSignal = CpuMpData->CpuData[Index3].StartupApSignal;
+ CpuMpData->CpuData[Index3].StartupApSignal =
+ CpuMpData->CpuData[Index1].StartupApSignal;
+ CpuMpData->CpuData[Index1].StartupApSignal = StartupApSignal;
}
}
@@ -325,13 +402,13 @@ ApInitializeSync (
CpuMpData = (CPU_MP_DATA *) Buffer;
//
- // Sync BSP's MTRR table to AP
+ // Load microcode on AP
//
- MtrrSetAllMtrrs (&CpuMpData->MtrrTable);
+ MicrocodeDetect (CpuMpData, FALSE);
//
- // Load microcode on AP
+ // Sync BSP's MTRR table to AP
//
- MicrocodeDetect (CpuMpData);
+ MtrrSetAllMtrrs (&CpuMpData->MtrrTable);
}
/**
@@ -352,16 +429,19 @@ GetProcessorNumber (
UINTN TotalProcessorNumber;
UINTN Index;
CPU_INFO_IN_HOB *CpuInfoInHob;
+ UINT32 CurrentApicId;
CpuInfoInHob = (CPU_INFO_IN_HOB *) (UINTN) CpuMpData->CpuInfoInHob;
TotalProcessorNumber = CpuMpData->CpuCount;
+ CurrentApicId = GetApicId ();
for (Index = 0; Index < TotalProcessorNumber; Index ++) {
- if (CpuInfoInHob[Index].ApicId == GetApicId ()) {
+ if (CpuInfoInHob[Index].ApicId == CurrentApicId) {
*ProcessorNumber = Index;
return EFI_SUCCESS;
}
}
+
return EFI_NOT_FOUND;
}
@@ -377,12 +457,14 @@ CollectProcessorCount (
IN CPU_MP_DATA *CpuMpData
)
{
+ UINTN Index;
+
//
// Send 1st broadcast IPI to APs to wakeup APs
//
CpuMpData->InitFlag = ApInitConfig;
CpuMpData->X2ApicEnable = FALSE;
- WakeUpAP (CpuMpData, TRUE, 0, NULL, NULL);
+ WakeUpAP (CpuMpData, TRUE, 0, NULL, NULL, TRUE);
CpuMpData->InitFlag = ApInitDone;
ASSERT (CpuMpData->CpuCount <= PcdGet32 (PcdCpuMaxLogicalProcessorNumber));
//
@@ -392,12 +474,18 @@ CollectProcessorCount (
CpuPause ();
}
+ if (CpuMpData->CpuCount > 255) {
+ //
+ // If there are more than 255 processor found, force to enable X2APIC
+ //
+ CpuMpData->X2ApicEnable = TRUE;
+ }
if (CpuMpData->X2ApicEnable) {
DEBUG ((DEBUG_INFO, "Force x2APIC mode!\n"));
//
// Wakeup all APs to enable x2APIC mode
//
- WakeUpAP (CpuMpData, TRUE, 0, ApFuncEnableX2Apic, NULL);
+ WakeUpAP (CpuMpData, TRUE, 0, ApFuncEnableX2Apic, NULL, TRUE);
//
// Wait for all known APs finished
//
@@ -408,6 +496,12 @@ CollectProcessorCount (
// Enable x2APIC on BSP
//
SetApicMode (LOCAL_APIC_MODE_X2APIC);
+ //
+ // Set BSP/Aps state to IDLE
+ //
+ for (Index = 0; Index < CpuMpData->CpuCount; Index++) {
+ SetApState (&CpuMpData->CpuData[Index], CpuStateIdle);
+ }
}
DEBUG ((DEBUG_INFO, "APIC MODE is %d\n", GetApicMode ()));
//
@@ -465,13 +559,13 @@ InitializeApData (
This function will be called from AP reset code if BSP uses WakeUpAP.
@param[in] ExchangeInfo Pointer to the MP exchange info buffer
- @param[in] NumApsExecuting Number of current executing AP
+ @param[in] ApIndex Number of current executing AP
**/
VOID
EFIAPI
ApWakeupFunction (
IN MP_CPU_EXCHANGE_INFO *ExchangeInfo,
- IN UINTN NumApsExecuting
+ IN UINTN ApIndex
)
{
CPU_MP_DATA *CpuMpData;
@@ -482,21 +576,32 @@ ApWakeupFunction (
volatile UINT32 *ApStartupSignalBuffer;
CPU_INFO_IN_HOB *CpuInfoInHob;
UINT64 ApTopOfStack;
+ UINTN CurrentApicMode;
//
// AP finished assembly code and begin to execute C code
//
CpuMpData = ExchangeInfo->CpuMpData;
- ProgramVirtualWireMode ();
+ //
+ // AP's local APIC settings will be lost after received INIT IPI
+ // We need to re-initialize them at here
+ //
+ ProgramVirtualWireMode ();
+ //
+ // Mask the LINT0 and LINT1 so that AP doesn't enter the system timer interrupt handler.
+ //
+ DisableLvtInterrupts ();
+ SyncLocalApicTimerSetting (CpuMpData);
+ CurrentApicMode = GetApicMode ();
while (TRUE) {
if (CpuMpData->InitFlag == ApInitConfig) {
//
// Add CPU number
//
InterlockedIncrement ((UINT32 *) &CpuMpData->CpuCount);
- ProcessorNumber = NumApsExecuting;
+ ProcessorNumber = ApIndex;
//
// This is first time AP wakeup, get BIST information from AP stack
//
@@ -507,7 +612,9 @@ ApWakeupFunction (
//
ApInitializeSync (CpuMpData);
//
- // Sync BSP's Control registers to APs
+ // CpuMpData->CpuData[0].VolatileRegisters is initialized based on BSP environment,
+ // to initialize AP in InitConfig path.
+ // NOTE: IDTR.BASE stored in CpuMpData->CpuData[0].VolatileRegisters points to a different IDT shared by all APs.
//
RestoreVolatileRegisters (&CpuMpData->CpuData[0].VolatileRegisters, FALSE);
InitializeApData (CpuMpData, ProcessorNumber, BistData, ApTopOfStack);
@@ -531,6 +638,13 @@ ApWakeupFunction (
// Restore AP's volatile registers saved
//
RestoreVolatileRegisters (&CpuMpData->CpuData[ProcessorNumber].VolatileRegisters, TRUE);
+ } else {
+ //
+ // The CPU driver might not flush TLB for APs on spot after updating
+ // page attributes. AP in mwait loop mode needs to take care of it when
+ // woken up.
+ //
+ CpuFlushTlb ();
}
if (GetApState (&CpuMpData->CpuData[ProcessorNumber]) == CpuStateReady) {
@@ -539,6 +653,10 @@ ApWakeupFunction (
if (Procedure != NULL) {
SetApState (&CpuMpData->CpuData[ProcessorNumber], CpuStateBusy);
//
+ // Enable source debugging on AP function
+ //
+ EnableDebugAgent ();
+ //
// Invoke AP function here
//
Procedure (Parameter);
@@ -553,11 +671,23 @@ ApWakeupFunction (
ApStartupSignalBuffer = CpuMpData->CpuData[ProcessorNumber].StartupApSignal;
CpuInfoInHob[ProcessorNumber].ApTopOfStack = CpuInfoInHob[CpuMpData->NewBspNumber].ApTopOfStack;
} else {
- //
- // Re-get the CPU APICID and Initial APICID
- //
- CpuInfoInHob[ProcessorNumber].ApicId = GetApicId ();
- CpuInfoInHob[ProcessorNumber].InitialApicId = GetInitialApicId ();
+ if (CpuInfoInHob[ProcessorNumber].ApicId != GetApicId () ||
+ CpuInfoInHob[ProcessorNumber].InitialApicId != GetInitialApicId ()) {
+ if (CurrentApicMode != GetApicMode ()) {
+ //
+ // If APIC mode change happened during AP function execution,
+ // we do not support APIC ID value changed.
+ //
+ ASSERT (FALSE);
+ CpuDeadLoop ();
+ } else {
+ //
+ // Re-get the CPU APICID and Initial APICID if they are changed
+ //
+ CpuInfoInHob[ProcessorNumber].ApicId = GetApicId ();
+ CpuInfoInHob[ProcessorNumber].InitialApicId = GetInitialApicId ();
+ }
+ }
}
}
SetApState (&CpuMpData->CpuData[ProcessorNumber], CpuStateFinished);
@@ -568,6 +698,7 @@ ApWakeupFunction (
// AP finished executing C code
//
InterlockedIncrement ((UINT32 *) &CpuMpData->FinishedCount);
+ InterlockedDecrement ((UINT32 *) &CpuMpData->MpCpuExchangeInfo->NumApsExecuting);
//
// Place AP is specified loop mode
@@ -656,6 +787,8 @@ FillExchangeInfoData (
)
{
volatile MP_CPU_EXCHANGE_INFO *ExchangeInfo;
+ UINTN Size;
+ IA32_SEGMENT_DESCRIPTOR *Selector;
ExchangeInfo = CpuMpData->MpCpuExchangeInfo;
ExchangeInfo->Lock = 0;
@@ -670,6 +803,7 @@ FillExchangeInfoData (
ExchangeInfo->Cr3 = AsmReadCr3 ();
ExchangeInfo->CFunction = (UINTN) ApWakeupFunction;
+ ExchangeInfo->ApIndex = 0;
ExchangeInfo->NumApsExecuting = 0;
ExchangeInfo->InitFlag = (UINTN) CpuMpData->InitFlag;
ExchangeInfo->CpuInfo = (CPU_INFO_IN_HOB *) (UINTN) CpuMpData->CpuInfoInHob;
@@ -677,11 +811,53 @@ FillExchangeInfoData (
ExchangeInfo->EnableExecuteDisable = IsBspExecuteDisableEnabled ();
+ ExchangeInfo->InitializeFloatingPointUnitsAddress = (UINTN)InitializeFloatingPointUnits;
+
//
// Get the BSP's data of GDT and IDT
//
AsmReadGdtr ((IA32_DESCRIPTOR *) &ExchangeInfo->GdtrProfile);
AsmReadIdtr ((IA32_DESCRIPTOR *) &ExchangeInfo->IdtrProfile);
+
+ //
+ // Find a 32-bit code segment
+ //
+ Selector = (IA32_SEGMENT_DESCRIPTOR *)ExchangeInfo->GdtrProfile.Base;
+ Size = ExchangeInfo->GdtrProfile.Limit + 1;
+ while (Size > 0) {
+ if (Selector->Bits.L == 0 && Selector->Bits.Type >= 8) {
+ ExchangeInfo->ModeTransitionSegment =
+ (UINT16)((UINTN)Selector - ExchangeInfo->GdtrProfile.Base);
+ break;
+ }
+ Selector += 1;
+ Size -= sizeof (IA32_SEGMENT_DESCRIPTOR);
+ }
+
+ //
+ // Copy all 32-bit code and 64-bit code into memory with type of
+ // EfiBootServicesCode to avoid page fault if NX memory protection is enabled.
+ //
+ if (CpuMpData->WakeupBufferHigh != 0) {
+ Size = CpuMpData->AddressMap.RendezvousFunnelSize -
+ CpuMpData->AddressMap.ModeTransitionOffset;
+ CopyMem (
+ (VOID *)CpuMpData->WakeupBufferHigh,
+ CpuMpData->AddressMap.RendezvousFunnelAddress +
+ CpuMpData->AddressMap.ModeTransitionOffset,
+ Size
+ );
+
+ ExchangeInfo->ModeTransitionMemory = (UINT32)CpuMpData->WakeupBufferHigh;
+ } else {
+ ExchangeInfo->ModeTransitionMemory = (UINT32)
+ (ExchangeInfo->BufferStart + CpuMpData->AddressMap.ModeTransitionOffset);
+ }
+
+ ExchangeInfo->ModeHighMemory = ExchangeInfo->ModeTransitionMemory +
+ (UINT32)ExchangeInfo->ModeOffset -
+ (UINT32)CpuMpData->AddressMap.ModeTransitionOffset;
+ ExchangeInfo->ModeHighSegment = (UINT16)ExchangeInfo->CodeSegment;
}
/**
@@ -699,6 +875,85 @@ TimedWaitForApFinish (
IN UINT32 TimeLimit
);
+/**
+ Get available system memory below 1MB by specified size.
+
+ @param[in] CpuMpData The pointer to CPU MP Data structure.
+**/
+VOID
+BackupAndPrepareWakeupBuffer(
+ IN CPU_MP_DATA *CpuMpData
+ )
+{
+ CopyMem (
+ (VOID *) CpuMpData->BackupBuffer,
+ (VOID *) CpuMpData->WakeupBuffer,
+ CpuMpData->BackupBufferSize
+ );
+ CopyMem (
+ (VOID *) CpuMpData->WakeupBuffer,
+ (VOID *) CpuMpData->AddressMap.RendezvousFunnelAddress,
+ CpuMpData->AddressMap.RendezvousFunnelSize
+ );
+}
+
+/**
+ Restore wakeup buffer data.
+
+ @param[in] CpuMpData The pointer to CPU MP Data structure.
+**/
+VOID
+RestoreWakeupBuffer(
+ IN CPU_MP_DATA *CpuMpData
+ )
+{
+ CopyMem (
+ (VOID *) CpuMpData->WakeupBuffer,
+ (VOID *) CpuMpData->BackupBuffer,
+ CpuMpData->BackupBufferSize
+ );
+}
+
+/**
+ Allocate reset vector buffer.
+
+ @param[in, out] CpuMpData The pointer to CPU MP Data structure.
+**/
+VOID
+AllocateResetVector (
+ IN OUT CPU_MP_DATA *CpuMpData
+ )
+{
+ UINTN ApResetVectorSize;
+
+ if (CpuMpData->WakeupBuffer == (UINTN) -1) {
+ ApResetVectorSize = CpuMpData->AddressMap.RendezvousFunnelSize +
+ sizeof (MP_CPU_EXCHANGE_INFO);
+
+ CpuMpData->WakeupBuffer = GetWakeupBuffer (ApResetVectorSize);
+ CpuMpData->MpCpuExchangeInfo = (MP_CPU_EXCHANGE_INFO *) (UINTN)
+ (CpuMpData->WakeupBuffer + CpuMpData->AddressMap.RendezvousFunnelSize);
+ CpuMpData->WakeupBufferHigh = GetModeTransitionBuffer (
+ CpuMpData->AddressMap.RendezvousFunnelSize -
+ CpuMpData->AddressMap.ModeTransitionOffset
+ );
+ }
+ BackupAndPrepareWakeupBuffer (CpuMpData);
+}
+
+/**
+ Free AP reset vector buffer.
+
+ @param[in] CpuMpData The pointer to CPU MP Data structure.
+**/
+VOID
+FreeResetVector (
+ IN CPU_MP_DATA *CpuMpData
+ )
+{
+ RestoreWakeupBuffer (CpuMpData);
+}
+
/**
This function will be called by BSP to wakeup AP.
@@ -708,6 +963,7 @@ TimedWaitForApFinish (
@param[in] ProcessorNumber The handle number of specified processor
@param[in] Procedure The function to be invoked by AP
@param[in] ProcedureArgument The argument to be passed into AP function
+ @param[in] WakeUpDisabledAps Whether need to wake up disabled APs in broadcast mode.
**/
VOID
WakeUpAP (
@@ -715,7 +971,8 @@ WakeUpAP (
IN BOOLEAN Broadcast,
IN UINTN ProcessorNumber,
IN EFI_AP_PROCEDURE Procedure, OPTIONAL
- IN VOID *ProcedureArgument OPTIONAL
+ IN VOID *ProcedureArgument, OPTIONAL
+ IN BOOLEAN WakeUpDisabledAps
)
{
volatile MP_CPU_EXCHANGE_INFO *ExchangeInfo;
@@ -727,12 +984,15 @@ WakeUpAP (
CpuMpData->FinishedCount = 0;
ResetVectorRequired = FALSE;
- if (CpuMpData->ApLoopMode == ApInHltLoop ||
+ if (CpuMpData->WakeUpByInitSipiSipi ||
CpuMpData->InitFlag != ApInitDone) {
ResetVectorRequired = TRUE;
AllocateResetVector (CpuMpData);
FillExchangeInfoData (CpuMpData);
- } else if (CpuMpData->ApLoopMode == ApInMwaitLoop) {
+ SaveLocalApicTimerSetting (CpuMpData);
+ }
+
+ if (CpuMpData->ApLoopMode == ApInMwaitLoop) {
//
// Get AP target C-state each time when waking up AP,
// for it maybe updated by platform again
@@ -746,6 +1006,15 @@ WakeUpAP (
for (Index = 0; Index < CpuMpData->CpuCount; Index++) {
if (Index != CpuMpData->BspNumber) {
CpuData = &CpuMpData->CpuData[Index];
+ //
+ // All AP(include disabled AP) will be woke up by INIT-SIPI-SIPI, but
+ // the AP procedure will be skipped for disabled AP because AP state
+ // is not CpuStateReady.
+ //
+ if (GetApState (CpuData) == CpuStateDisabled && !WakeUpDisabledAps) {
+ continue;
+ }
+
CpuData->ApFunction = (UINTN) Procedure;
CpuData->ApFunctionArgument = (UINTN) ProcedureArgument;
SetApState (CpuData, CpuStateReady);
@@ -762,13 +1031,24 @@ WakeUpAP (
}
if (CpuMpData->InitFlag == ApInitConfig) {
//
- // Wait for all potential APs waken up in one specified period
+ // Here support two methods to collect AP count through adjust
+ // PcdCpuApInitTimeOutInMicroSeconds values.
+ //
+ // one way is set a value to just let the first AP to start the
+ // initialization, then through the later while loop to wait all Aps
+ // finsh the initialization.
+ // The other way is set a value to let all APs finished the initialzation.
+ // In this case, the later while loop is useless.
//
TimedWaitForApFinish (
CpuMpData,
PcdGet32 (PcdCpuMaxLogicalProcessorNumber) - 1,
PcdGet32 (PcdCpuApInitTimeOutInMicroSeconds)
);
+
+ while (CpuMpData->MpCpuExchangeInfo->NumApsExecuting != 0) {
+ CpuPause();
+ }
} else {
//
// Wait all APs waken up if this is not the 1st broadcast of SIPI
@@ -806,6 +1086,13 @@ WakeUpAP (
if (ResetVectorRequired) {
FreeResetVector (CpuMpData);
}
+
+ //
+ // After one round of Wakeup Ap actions, need to re-sync ApLoopMode with
+ // WakeUpByInitSipiSipi flag. WakeUpByInitSipiSipi flag maybe changed by
+ // S3SmmInitDone Ppi.
+ //
+ CpuMpData->WakeUpByInitSipiSipi = (CpuMpData->ApLoopMode == ApInHltLoop);
}
/**
@@ -829,6 +1116,9 @@ CalculateTimeout (
OUT UINT64 *CurrentTime
)
{
+ UINT64 TimeoutInSeconds;
+ UINT64 TimestampCounterFreq;
+
//
// Read the current value of the performance counter
//
@@ -844,16 +1134,36 @@ CalculateTimeout (
//
// GetPerformanceCounterProperties () returns the timestamp counter's frequency
- // in Hz. So multiply the return value with TimeoutInMicroseconds and then divide
- // it by 1,000,000, to get the number of ticks for the timeout value.
- //
- return DivU64x32 (
- MultU64x64 (
- GetPerformanceCounterProperties (NULL, NULL),
- TimeoutInMicroseconds
- ),
- 1000000
- );
+ // in Hz.
+ //
+ TimestampCounterFreq = GetPerformanceCounterProperties (NULL, NULL);
+
+ //
+ // Check the potential overflow before calculate the number of ticks for the timeout value.
+ //
+ if (DivU64x64Remainder (MAX_UINT64, TimeoutInMicroseconds, NULL) < TimestampCounterFreq) {
+ //
+ // Convert microseconds into seconds if direct multiplication overflows
+ //
+ TimeoutInSeconds = DivU64x32 (TimeoutInMicroseconds, 1000000);
+ //
+ // Assertion if the final tick count exceeds MAX_UINT64
+ //
+ ASSERT (DivU64x64Remainder (MAX_UINT64, TimeoutInSeconds, NULL) >= TimestampCounterFreq);
+ return MultU64x64 (TimestampCounterFreq, TimeoutInSeconds);
+ } else {
+ //
+ // No overflow case, multiply the return value with TimeoutInMicroseconds and then divide
+ // it by 1,000,000, to get the number of ticks for the timeout value.
+ //
+ return DivU64x32 (
+ MultU64x64 (
+ TimestampCounterFreq,
+ TimeoutInMicroseconds
+ ),
+ 1000000
+ );
+ }
}
/**
@@ -984,7 +1294,7 @@ ResetProcessorToIdleState (
CpuMpData = GetCpuMpData ();
CpuMpData->InitFlag = ApInitReconfig;
- WakeUpAP (CpuMpData, FALSE, ProcessorNumber, NULL, NULL);
+ WakeUpAP (CpuMpData, FALSE, ProcessorNumber, NULL, NULL, TRUE);
while (CpuMpData->FinishedCount < 1) {
CpuPause ();
}
@@ -1047,9 +1357,9 @@ CheckThisAP (
CpuData = &CpuMpData->CpuData[ProcessorNumber];
//
- // Check the CPU state of AP. If it is CpuStateFinished, then the AP has finished its task.
+ // Check the CPU state of AP. If it is CpuStateIdle, then the AP has finished its task.
// Only BSP and corresponding AP access this unit of CPU Data. This means the AP will not modify the
- // value of state after setting the it to CpuStateFinished, so BSP can safely make use of its value.
+ // value of state after setting the it to CpuStateIdle, so BSP can safely make use of its value.
//
//
// If the AP finishes for StartupThisAP(), return EFI_SUCCESS.
@@ -1115,12 +1425,12 @@ CheckAllAPs (
CpuData = &CpuMpData->CpuData[ProcessorNumber];
//
- // Check the CPU state of AP. If it is CpuStateFinished, then the AP has finished its task.
+ // Check the CPU state of AP. If it is CpuStateIdle, then the AP has finished its task.
// Only BSP and corresponding AP access this unit of CPU Data. This means the AP will not modify the
- // value of state after setting the it to CpuStateFinished, so BSP can safely make use of its value.
+ // value of state after setting the it to CpuStateIdle, so BSP can safely make use of its value.
//
if (GetApState(CpuData) == CpuStateFinished) {
- CpuMpData->RunningCount ++;
+ CpuMpData->RunningCount --;
CpuMpData->CpuData[ProcessorNumber].Waiting = FALSE;
SetApState(CpuData, CpuStateIdle);
@@ -1136,7 +1446,8 @@ CheckAllAPs (
FALSE,
(UINT32) NextProcessorNumber,
CpuMpData->Procedure,
- CpuMpData->ProcArguments
+ CpuMpData->ProcArguments,
+ TRUE
);
}
}
@@ -1146,7 +1457,7 @@ CheckAllAPs (
//
// If all APs finish, return EFI_SUCCESS.
//
- if (CpuMpData->RunningCount == CpuMpData->StartCount) {
+ if (CpuMpData->RunningCount == 0) {
return EFI_SUCCESS;
}
@@ -1163,7 +1474,7 @@ CheckAllAPs (
//
if (CpuMpData->FailedCpuList != NULL) {
*CpuMpData->FailedCpuList =
- AllocatePool ((CpuMpData->StartCount - CpuMpData->FinishedCount + 1) * sizeof (UINTN));
+ AllocatePool ((CpuMpData->RunningCount + 1) * sizeof (UINTN));
ASSERT (*CpuMpData->FailedCpuList != NULL);
}
ListIndex = 0;
@@ -1215,6 +1526,7 @@ MpInitLibInitialize (
UINT32 MaxLogicalProcessorNumber;
UINT32 ApStackSize;
MP_ASSEMBLY_ADDRESS_MAP AddressMap;
+ CPU_VOLATILE_REGISTERS VolatileRegisters;
UINTN BufferSize;
UINT32 MonitorFilterSize;
VOID *MpBuffer;
@@ -1225,6 +1537,8 @@ MpInitLibInitialize (
UINTN Index;
UINTN ApResetVectorSize;
UINTN BackupBufferAddr;
+ UINTN ApIdtBase;
+ VOID *MicrocodePatchInRam;
OldCpuMpData = GetCpuMpDataFromGuidedHob ();
if (OldCpuMpData == NULL) {
@@ -1239,24 +1553,52 @@ MpInitLibInitialize (
ApStackSize = PcdGet32(PcdCpuApStackSize);
ApLoopMode = GetApLoopMode (&MonitorFilterSize);
+ //
+ // Save BSP's Control registers for APs.
+ //
+ SaveVolatileRegisters (&VolatileRegisters);
+
BufferSize = ApStackSize * MaxLogicalProcessorNumber;
BufferSize += MonitorFilterSize * MaxLogicalProcessorNumber;
- BufferSize += sizeof (CPU_MP_DATA);
BufferSize += ApResetVectorSize;
+ BufferSize = ALIGN_VALUE (BufferSize, 8);
+ BufferSize += VolatileRegisters.Idtr.Limit + 1;
+ BufferSize += sizeof (CPU_MP_DATA);
BufferSize += (sizeof (CPU_AP_DATA) + sizeof (CPU_INFO_IN_HOB))* MaxLogicalProcessorNumber;
MpBuffer = AllocatePages (EFI_SIZE_TO_PAGES (BufferSize));
ASSERT (MpBuffer != NULL);
ZeroMem (MpBuffer, BufferSize);
Buffer = (UINTN) MpBuffer;
+ //
+ // The layout of the Buffer is as below:
+ //
+ // +--------------------+ <-- Buffer
+ // AP Stacks (N)
+ // +--------------------+ <-- MonitorBuffer
+ // AP Monitor Filters (N)
+ // +--------------------+ <-- BackupBufferAddr (CpuMpData->BackupBuffer)
+ // Backup Buffer
+ // +--------------------+
+ // Padding
+ // +--------------------+ <-- ApIdtBase (8-byte boundary)
+ // AP IDT All APs share one separate IDT. So AP can get address of CPU_MP_DATA from IDT Base.
+ // +--------------------+ <-- CpuMpData
+ // CPU_MP_DATA
+ // +--------------------+ <-- CpuMpData->CpuData
+ // CPU_AP_DATA (N)
+ // +--------------------+ <-- CpuMpData->CpuInfoInHob
+ // CPU_INFO_IN_HOB (N)
+ // +--------------------+
+ //
MonitorBuffer = (UINT8 *) (Buffer + ApStackSize * MaxLogicalProcessorNumber);
BackupBufferAddr = (UINTN) MonitorBuffer + MonitorFilterSize * MaxLogicalProcessorNumber;
- CpuMpData = (CPU_MP_DATA *) (BackupBufferAddr + ApResetVectorSize);
+ ApIdtBase = ALIGN_VALUE (BackupBufferAddr + ApResetVectorSize, 8);
+ CpuMpData = (CPU_MP_DATA *) (ApIdtBase + VolatileRegisters.Idtr.Limit + 1);
CpuMpData->Buffer = Buffer;
CpuMpData->CpuApStackSize = ApStackSize;
CpuMpData->BackupBuffer = BackupBufferAddr;
CpuMpData->BackupBufferSize = ApResetVectorSize;
- CpuMpData->SaveRestoreFlag = FALSE;
CpuMpData->WakeupBuffer = (UINTN) -1;
CpuMpData->CpuCount = 1;
CpuMpData->BspNumber = 0;
@@ -1264,15 +1606,61 @@ MpInitLibInitialize (
CpuMpData->SwitchBspFlag = FALSE;
CpuMpData->CpuData = (CPU_AP_DATA *) (CpuMpData + 1);
CpuMpData->CpuInfoInHob = (UINT64) (UINTN) (CpuMpData->CpuData + MaxLogicalProcessorNumber);
+ CpuMpData->MicrocodePatchRegionSize = PcdGet64 (PcdCpuMicrocodePatchRegionSize);
+ //
+ // If platform has more than one CPU, relocate microcode to memory to reduce
+ // loading microcode time.
+ //
+ MicrocodePatchInRam = NULL;
+ if (MaxLogicalProcessorNumber > 1) {
+ MicrocodePatchInRam = AllocatePages (
+ EFI_SIZE_TO_PAGES (
+ (UINTN)CpuMpData->MicrocodePatchRegionSize
+ )
+ );
+ }
+ if (MicrocodePatchInRam == NULL) {
+ //
+ // there is only one processor, or no microcode patch is available, or
+ // memory allocation failed
+ //
+ CpuMpData->MicrocodePatchAddress = PcdGet64 (PcdCpuMicrocodePatchAddress);
+ } else {
+ //
+ // there are multiple processors, and a microcode patch is available, and
+ // memory allocation succeeded
+ //
+ CopyMem (
+ MicrocodePatchInRam,
+ (VOID *)(UINTN)PcdGet64 (PcdCpuMicrocodePatchAddress),
+ (UINTN)CpuMpData->MicrocodePatchRegionSize
+ );
+ CpuMpData->MicrocodePatchAddress = (UINTN)MicrocodePatchInRam;
+ }
+
InitializeSpinLock(&CpuMpData->MpLock);
+
+ //
+ // Make sure no memory usage outside of the allocated buffer.
+ //
+ ASSERT ((CpuMpData->CpuInfoInHob + sizeof (CPU_INFO_IN_HOB) * MaxLogicalProcessorNumber) ==
+ Buffer + BufferSize);
+
+ //
+ // Duplicate BSP's IDT to APs.
+ // All APs share one separate IDT. So AP can get the address of CpuMpData by using IDTR.BASE + IDTR.LIMIT + 1
//
- // Save BSP's Control registers to APs
+ CopyMem ((VOID *)ApIdtBase, (VOID *)VolatileRegisters.Idtr.Base, VolatileRegisters.Idtr.Limit + 1);
+ VolatileRegisters.Idtr.Base = ApIdtBase;
//
- SaveVolatileRegisters (&CpuMpData->CpuData[0].VolatileRegisters);
+ // Don't pass BSP's TR to APs to avoid AP init failure.
+ //
+ VolatileRegisters.Tr = 0;
+ CopyMem (&CpuMpData->CpuData[0].VolatileRegisters, &VolatileRegisters, sizeof (VolatileRegisters));
//
// Set BSP basic information
//
- InitializeApData (CpuMpData, 0, 0, CpuMpData->Buffer);
+ InitializeApData (CpuMpData, 0, 0, CpuMpData->Buffer + ApStackSize);
//
// Save assembly code information
//
@@ -1282,6 +1670,9 @@ MpInitLibInitialize (
//
CpuMpData->ApLoopMode = ApLoopMode;
DEBUG ((DEBUG_INFO, "AP Loop Mode is %d\n", CpuMpData->ApLoopMode));
+
+ CpuMpData->WakeUpByInitSipiSipi = (CpuMpData->ApLoopMode == ApInHltLoop);
+
//
// Set up APs wakeup signal buffer
//
@@ -1292,11 +1683,15 @@ MpInitLibInitialize (
//
// Load Microcode on BSP
//
- MicrocodeDetect (CpuMpData);
+ MicrocodeDetect (CpuMpData, TRUE);
//
// Store BSP's MTRR setting
//
MtrrGetAllMtrrs (&CpuMpData->MtrrTable);
+ //
+ // Enable the local APIC for Virtual Wire Mode.
+ //
+ ProgramVirtualWireMode ();
if (OldCpuMpData == NULL) {
if (MaxLogicalProcessorNumber > 1) {
@@ -1317,22 +1712,18 @@ MpInitLibInitialize (
CpuInfoInHob = (CPU_INFO_IN_HOB *) (UINTN) CpuMpData->CpuInfoInHob;
for (Index = 0; Index < CpuMpData->CpuCount; Index++) {
InitializeSpinLock(&CpuMpData->CpuData[Index].ApLock);
- if (CpuInfoInHob[Index].InitialApicId >= 255) {
+ if (CpuInfoInHob[Index].InitialApicId >= 255 || Index > 254) {
CpuMpData->X2ApicEnable = TRUE;
}
CpuMpData->CpuData[Index].CpuHealthy = (CpuInfoInHob[Index].Health == 0)? TRUE:FALSE;
CpuMpData->CpuData[Index].ApFunction = 0;
- CopyMem (
- &CpuMpData->CpuData[Index].VolatileRegisters,
- &CpuMpData->CpuData[0].VolatileRegisters,
- sizeof (CPU_VOLATILE_REGISTERS)
- );
+ CopyMem (&CpuMpData->CpuData[Index].VolatileRegisters, &VolatileRegisters, sizeof (CPU_VOLATILE_REGISTERS));
}
if (MaxLogicalProcessorNumber > 1) {
//
// Wakeup APs to do some AP initialize sync
//
- WakeUpAP (CpuMpData, TRUE, 0, ApInitializeSync, CpuMpData);
+ WakeUpAP (CpuMpData, TRUE, 0, ApInitializeSync, CpuMpData, TRUE);
//
// Wait for all APs finished initialization
//
@@ -1441,7 +1832,7 @@ MpInitLibGetProcessorInfo (
enabled AP. Otherwise, it will be disabled.
@retval EFI_SUCCESS BSP successfully switched.
- @retval others Failed to switch BSP.
+ @retval others Failed to switch BSP.
**/
EFI_STATUS
@@ -1454,6 +1845,27 @@ SwitchBSPWorker (
UINTN CallerNumber;
CPU_STATE State;
MSR_IA32_APIC_BASE_REGISTER ApicBaseMsr;
+ BOOLEAN OldInterruptState;
+ BOOLEAN OldTimerInterruptState;
+
+ //
+ // Save and Disable Local APIC timer interrupt
+ //
+ OldTimerInterruptState = GetApicTimerInterruptState ();
+ DisableApicTimerInterrupt ();
+ //
+ // Before send both BSP and AP to a procedure to exchange their roles,
+ // interrupt must be disabled. This is because during the exchange role
+ // process, 2 CPU may use 1 stack. If interrupt happens, the stack will
+ // be corrupted, since interrupt return address will be pushed to stack
+ // by hardware.
+ //
+ OldInterruptState = SaveAndDisableInterrupts ();
+
+ //
+ // Mask LINT0 & LINT1 for the old BSP
+ //
+ DisableLvtInterrupts ();
CpuMpData = GetCpuMpData ();
@@ -1462,7 +1874,7 @@ SwitchBSPWorker (
//
MpInitLibWhoAmI (&CallerNumber);
if (CallerNumber != CpuMpData->BspNumber) {
- return EFI_SUCCESS;
+ return EFI_DEVICE_ERROR;
}
if (ProcessorNumber >= CpuMpData->CpuCount) {
@@ -1506,7 +1918,7 @@ SwitchBSPWorker (
//
// Need to wakeUp AP (future BSP).
//
- WakeUpAP (CpuMpData, FALSE, ProcessorNumber, FutureBSPProc, CpuMpData);
+ WakeUpAP (CpuMpData, FALSE, ProcessorNumber, FutureBSPProc, CpuMpData, TRUE);
AsmExchangeRole (&CpuMpData->BSPInfo, &CpuMpData->APInfo);
@@ -1516,6 +1928,7 @@ SwitchBSPWorker (
ApicBaseMsr.Uint64 = AsmReadMsr64 (MSR_IA32_APIC_BASE);
ApicBaseMsr.Bits.BSP = 1;
AsmWriteMsr64 (MSR_IA32_APIC_BASE, ApicBaseMsr.Uint64);
+ ProgramVirtualWireMode ();
//
// Wait for old BSP finished AP task
@@ -1530,12 +1943,23 @@ SwitchBSPWorker (
//
if (!EnableOldBSP) {
SetApState (&CpuMpData->CpuData[CallerNumber], CpuStateDisabled);
+ } else {
+ SetApState (&CpuMpData->CpuData[CallerNumber], CpuStateIdle);
}
//
// Save new BSP number
//
CpuMpData->BspNumber = (UINT32) ProcessorNumber;
+ //
+ // Restore interrupt state.
+ //
+ SetInterruptState (OldInterruptState);
+
+ if (OldTimerInterruptState) {
+ EnableApicTimerInterrupt ();
+ }
+
return EFI_SUCCESS;
}
@@ -1584,7 +2008,7 @@ EnableDisableApWorker (
if (!EnableAP) {
SetApState (&CpuMpData->CpuData[ProcessorNumber], CpuStateDisabled);
} else {
- SetApState (&CpuMpData->CpuData[ProcessorNumber], CpuStateIdle);
+ ResetProcessorToIdleState (ProcessorNumber);
}
if (HealthFlag != NULL) {
@@ -1800,7 +2224,7 @@ StartupAllAPsWorker (
return EFI_NOT_STARTED;
}
- CpuMpData->StartCount = 0;
+ CpuMpData->RunningCount = 0;
for (ProcessorNumber = 0; ProcessorNumber < ProcessorCount; ProcessorNumber++) {
CpuData = &CpuMpData->CpuData[ProcessorNumber];
CpuData->Waiting = FALSE;
@@ -1810,7 +2234,7 @@ StartupAllAPsWorker (
// Mark this processor as responsible for current calling.
//
CpuData->Waiting = TRUE;
- CpuMpData->StartCount++;
+ CpuMpData->RunningCount++;
}
}
}
@@ -1819,7 +2243,6 @@ StartupAllAPsWorker (
CpuMpData->ProcArguments = ProcedureArgument;
CpuMpData->SingleThread = SingleThread;
CpuMpData->FinishedCount = 0;
- CpuMpData->RunningCount = 0;
CpuMpData->FailedCpuList = FailedCpuList;
CpuMpData->ExpectedTime = CalculateTimeout (
TimeoutInMicroseconds,
@@ -1829,14 +2252,14 @@ StartupAllAPsWorker (
CpuMpData->WaitEvent = WaitEvent;
if (!SingleThread) {
- WakeUpAP (CpuMpData, TRUE, 0, Procedure, ProcedureArgument);
+ WakeUpAP (CpuMpData, TRUE, 0, Procedure, ProcedureArgument, FALSE);
} else {
for (ProcessorNumber = 0; ProcessorNumber < ProcessorCount; ProcessorNumber++) {
if (ProcessorNumber == CallerNumber) {
continue;
}
if (CpuMpData->CpuData[ProcessorNumber].Waiting) {
- WakeUpAP (CpuMpData, FALSE, ProcessorNumber, Procedure, ProcedureArgument);
+ WakeUpAP (CpuMpData, FALSE, ProcessorNumber, Procedure, ProcedureArgument, TRUE);
break;
}
}
@@ -1948,7 +2371,7 @@ StartupThisAPWorker (
CpuData->ExpectedTime = CalculateTimeout (TimeoutInMicroseconds, &CpuData->CurrentTime);
CpuData->TotalTime = 0;
- WakeUpAP (CpuMpData, FALSE, ProcessorNumber, Procedure, ProcedureArgument);
+ WakeUpAP (CpuMpData, FALSE, ProcessorNumber, Procedure, ProcedureArgument, TRUE);
//
// If WaitEvent is NULL, execute in blocking mode.
@@ -1987,41 +2410,3 @@ GetCpuMpDataFromGuidedHob (
return CpuMpData;
}
-/**
- Get available system memory below 1MB by specified size.
-
- @param[in] CpuMpData The pointer to CPU MP Data structure.
-**/
-VOID
-BackupAndPrepareWakeupBuffer(
- IN CPU_MP_DATA *CpuMpData
- )
-{
- CopyMem (
- (VOID *) CpuMpData->BackupBuffer,
- (VOID *) CpuMpData->WakeupBuffer,
- CpuMpData->BackupBufferSize
- );
- CopyMem (
- (VOID *) CpuMpData->WakeupBuffer,
- (VOID *) CpuMpData->AddressMap.RendezvousFunnelAddress,
- CpuMpData->AddressMap.RendezvousFunnelSize
- );
-}
-
-/**
- Restore wakeup buffer data.
-
- @param[in] CpuMpData The pointer to CPU MP Data structure.
-**/
-VOID
-RestoreWakeupBuffer(
- IN CPU_MP_DATA *CpuMpData
- )
-{
- CopyMem (
- (VOID *) CpuMpData->WakeupBuffer,
- (VOID *) CpuMpData->BackupBuffer,
- CpuMpData->BackupBufferSize
- );
-}