2 CPU MP Initialize Library common functions.
4 Copyright (c) 2016 - 2020, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2020, AMD Inc. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
12 #include <Library/VmgExitLib.h>
13 #include <Register/Amd/Fam17Msr.h>
14 #include <Register/Amd/Ghcb.h>
16 EFI_GUID mCpuInitMpLibHobGuid
= CPU_INIT_MP_LIB_HOB_GUID
;
20 The function will check if BSP Execute Disable is enabled.
22 DxeIpl may have enabled Execute Disable for BSP, APs need to
23 get the status and sync up the settings.
24 If BSP's CR0.Paging is not set, BSP execute Disble feature is
27 @retval TRUE BSP Execute Disable is enabled.
28 @retval FALSE BSP Execute Disable is not enabled.
31 IsBspExecuteDisableEnabled (
36 CPUID_EXTENDED_CPU_SIG_EDX Edx
;
37 MSR_IA32_EFER_REGISTER EferMsr
;
42 Cr0
.UintN
= AsmReadCr0 ();
43 if (Cr0
.Bits
.PG
!= 0) {
45 // If CR0 Paging bit is set
47 AsmCpuid (CPUID_EXTENDED_FUNCTION
, &Eax
, NULL
, NULL
, NULL
);
48 if (Eax
>= CPUID_EXTENDED_CPU_SIG
) {
49 AsmCpuid (CPUID_EXTENDED_CPU_SIG
, NULL
, NULL
, NULL
, &Edx
.Uint32
);
52 // Bit 20: Execute Disable Bit available.
54 if (Edx
.Bits
.NX
!= 0) {
55 EferMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_EFER
);
58 // Bit 11: Execute Disable Bit enable.
60 if (EferMsr
.Bits
.NXE
!= 0) {
71 Worker function for SwitchBSP().
73 Worker function for SwitchBSP(), assigned to the AP which is intended
76 @param[in] Buffer Pointer to CPU MP Data
84 CPU_MP_DATA
*DataInHob
;
86 DataInHob
= (CPU_MP_DATA
*) Buffer
;
87 AsmExchangeRole (&DataInHob
->APInfo
, &DataInHob
->BSPInfo
);
91 Get the Application Processors state.
93 @param[in] CpuData The pointer to CPU_AP_DATA of specified AP
99 IN CPU_AP_DATA
*CpuData
102 return CpuData
->State
;
106 Set the Application Processors state.
108 @param[in] CpuData The pointer to CPU_AP_DATA of specified AP
109 @param[in] State The AP status
113 IN CPU_AP_DATA
*CpuData
,
117 AcquireSpinLock (&CpuData
->ApLock
);
118 CpuData
->State
= State
;
119 ReleaseSpinLock (&CpuData
->ApLock
);
123 Save BSP's local APIC timer setting.
125 @param[in] CpuMpData Pointer to CPU MP Data
128 SaveLocalApicTimerSetting (
129 IN CPU_MP_DATA
*CpuMpData
133 // Record the current local APIC timer setting of BSP
136 &CpuMpData
->DivideValue
,
137 &CpuMpData
->PeriodicMode
,
140 CpuMpData
->CurrentTimerCount
= GetApicTimerCurrentCount ();
141 CpuMpData
->TimerInterruptState
= GetApicTimerInterruptState ();
145 Sync local APIC timer setting from BSP to AP.
147 @param[in] CpuMpData Pointer to CPU MP Data
150 SyncLocalApicTimerSetting (
151 IN CPU_MP_DATA
*CpuMpData
155 // Sync local APIC timer setting from BSP to AP
157 InitializeApicTimer (
158 CpuMpData
->DivideValue
,
159 CpuMpData
->CurrentTimerCount
,
160 CpuMpData
->PeriodicMode
,
164 // Disable AP's local APIC timer interrupt
166 DisableApicTimerInterrupt ();
170 Save the volatile registers required to be restored following INIT IPI.
172 @param[out] VolatileRegisters Returns buffer saved the volatile resisters
175 SaveVolatileRegisters (
176 OUT CPU_VOLATILE_REGISTERS
*VolatileRegisters
179 CPUID_VERSION_INFO_EDX VersionInfoEdx
;
181 VolatileRegisters
->Cr0
= AsmReadCr0 ();
182 VolatileRegisters
->Cr3
= AsmReadCr3 ();
183 VolatileRegisters
->Cr4
= AsmReadCr4 ();
185 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, NULL
, &VersionInfoEdx
.Uint32
);
186 if (VersionInfoEdx
.Bits
.DE
!= 0) {
188 // If processor supports Debugging Extensions feature
189 // by CPUID.[EAX=01H]:EDX.BIT2
191 VolatileRegisters
->Dr0
= AsmReadDr0 ();
192 VolatileRegisters
->Dr1
= AsmReadDr1 ();
193 VolatileRegisters
->Dr2
= AsmReadDr2 ();
194 VolatileRegisters
->Dr3
= AsmReadDr3 ();
195 VolatileRegisters
->Dr6
= AsmReadDr6 ();
196 VolatileRegisters
->Dr7
= AsmReadDr7 ();
199 AsmReadGdtr (&VolatileRegisters
->Gdtr
);
200 AsmReadIdtr (&VolatileRegisters
->Idtr
);
201 VolatileRegisters
->Tr
= AsmReadTr ();
205 Restore the volatile registers following INIT IPI.
207 @param[in] VolatileRegisters Pointer to volatile resisters
208 @param[in] IsRestoreDr TRUE: Restore DRx if supported
209 FALSE: Do not restore DRx
212 RestoreVolatileRegisters (
213 IN CPU_VOLATILE_REGISTERS
*VolatileRegisters
,
214 IN BOOLEAN IsRestoreDr
217 CPUID_VERSION_INFO_EDX VersionInfoEdx
;
218 IA32_TSS_DESCRIPTOR
*Tss
;
220 AsmWriteCr3 (VolatileRegisters
->Cr3
);
221 AsmWriteCr4 (VolatileRegisters
->Cr4
);
222 AsmWriteCr0 (VolatileRegisters
->Cr0
);
225 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, NULL
, &VersionInfoEdx
.Uint32
);
226 if (VersionInfoEdx
.Bits
.DE
!= 0) {
228 // If processor supports Debugging Extensions feature
229 // by CPUID.[EAX=01H]:EDX.BIT2
231 AsmWriteDr0 (VolatileRegisters
->Dr0
);
232 AsmWriteDr1 (VolatileRegisters
->Dr1
);
233 AsmWriteDr2 (VolatileRegisters
->Dr2
);
234 AsmWriteDr3 (VolatileRegisters
->Dr3
);
235 AsmWriteDr6 (VolatileRegisters
->Dr6
);
236 AsmWriteDr7 (VolatileRegisters
->Dr7
);
240 AsmWriteGdtr (&VolatileRegisters
->Gdtr
);
241 AsmWriteIdtr (&VolatileRegisters
->Idtr
);
242 if (VolatileRegisters
->Tr
!= 0 &&
243 VolatileRegisters
->Tr
< VolatileRegisters
->Gdtr
.Limit
) {
244 Tss
= (IA32_TSS_DESCRIPTOR
*)(VolatileRegisters
->Gdtr
.Base
+
245 VolatileRegisters
->Tr
);
246 if (Tss
->Bits
.P
== 1) {
247 Tss
->Bits
.Type
&= 0xD; // 1101 - Clear busy bit just in case
248 AsmWriteTr (VolatileRegisters
->Tr
);
254 Detect whether Mwait-monitor feature is supported.
256 @retval TRUE Mwait-monitor feature is supported.
257 @retval FALSE Mwait-monitor feature is not supported.
264 CPUID_VERSION_INFO_ECX VersionInfoEcx
;
266 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, &VersionInfoEcx
.Uint32
, NULL
);
267 return (VersionInfoEcx
.Bits
.MONITOR
== 1) ? TRUE
: FALSE
;
273 @param[out] MonitorFilterSize Returns the largest monitor-line size in bytes.
275 @return The AP loop mode.
279 OUT UINT32
*MonitorFilterSize
283 CPUID_MONITOR_MWAIT_EBX MonitorMwaitEbx
;
285 ASSERT (MonitorFilterSize
!= NULL
);
287 ApLoopMode
= PcdGet8 (PcdCpuApLoopMode
);
288 ASSERT (ApLoopMode
>= ApInHltLoop
&& ApLoopMode
<= ApInRunLoop
);
289 if (ApLoopMode
== ApInMwaitLoop
) {
290 if (!IsMwaitSupport ()) {
292 // If processor does not support MONITOR/MWAIT feature,
293 // force AP in Hlt-loop mode
295 ApLoopMode
= ApInHltLoop
;
298 if (PcdGetBool (PcdSevEsIsEnabled
)) {
300 // For SEV-ES, force AP in Hlt-loop mode in order to use the GHCB
301 // protocol for starting APs
303 ApLoopMode
= ApInHltLoop
;
307 if (ApLoopMode
!= ApInMwaitLoop
) {
308 *MonitorFilterSize
= sizeof (UINT32
);
311 // CPUID.[EAX=05H]:EBX.BIT0-15: Largest monitor-line size in bytes
312 // CPUID.[EAX=05H].EDX: C-states supported using MWAIT
314 AsmCpuid (CPUID_MONITOR_MWAIT
, NULL
, &MonitorMwaitEbx
.Uint32
, NULL
, NULL
);
315 *MonitorFilterSize
= MonitorMwaitEbx
.Bits
.LargestMonitorLineSize
;
322 Sort the APIC ID of all processors.
324 This function sorts the APIC ID of all processors so that processor number is
325 assigned in the ascending order of APIC ID which eases MP debugging.
327 @param[in] CpuMpData Pointer to PEI CPU MP Data
331 IN CPU_MP_DATA
*CpuMpData
338 CPU_INFO_IN_HOB CpuInfo
;
340 CPU_INFO_IN_HOB
*CpuInfoInHob
;
341 volatile UINT32
*StartupApSignal
;
343 ApCount
= CpuMpData
->CpuCount
- 1;
344 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
346 for (Index1
= 0; Index1
< ApCount
; Index1
++) {
349 // Sort key is the hardware default APIC ID
351 ApicId
= CpuInfoInHob
[Index1
].ApicId
;
352 for (Index2
= Index1
+ 1; Index2
<= ApCount
; Index2
++) {
353 if (ApicId
> CpuInfoInHob
[Index2
].ApicId
) {
355 ApicId
= CpuInfoInHob
[Index2
].ApicId
;
358 if (Index3
!= Index1
) {
359 CopyMem (&CpuInfo
, &CpuInfoInHob
[Index3
], sizeof (CPU_INFO_IN_HOB
));
361 &CpuInfoInHob
[Index3
],
362 &CpuInfoInHob
[Index1
],
363 sizeof (CPU_INFO_IN_HOB
)
365 CopyMem (&CpuInfoInHob
[Index1
], &CpuInfo
, sizeof (CPU_INFO_IN_HOB
));
368 // Also exchange the StartupApSignal.
370 StartupApSignal
= CpuMpData
->CpuData
[Index3
].StartupApSignal
;
371 CpuMpData
->CpuData
[Index3
].StartupApSignal
=
372 CpuMpData
->CpuData
[Index1
].StartupApSignal
;
373 CpuMpData
->CpuData
[Index1
].StartupApSignal
= StartupApSignal
;
378 // Get the processor number for the BSP
380 ApicId
= GetInitialApicId ();
381 for (Index1
= 0; Index1
< CpuMpData
->CpuCount
; Index1
++) {
382 if (CpuInfoInHob
[Index1
].ApicId
== ApicId
) {
383 CpuMpData
->BspNumber
= (UINT32
) Index1
;
391 Enable x2APIC mode on APs.
393 @param[in, out] Buffer Pointer to private data buffer.
401 SetApicMode (LOCAL_APIC_MODE_X2APIC
);
407 @param[in, out] Buffer Pointer to private data buffer.
415 CPU_MP_DATA
*CpuMpData
;
416 UINTN ProcessorNumber
;
419 CpuMpData
= (CPU_MP_DATA
*) Buffer
;
420 Status
= GetProcessorNumber (CpuMpData
, &ProcessorNumber
);
421 ASSERT_EFI_ERROR (Status
);
423 // Load microcode on AP
425 MicrocodeDetect (CpuMpData
, ProcessorNumber
);
427 // Sync BSP's MTRR table to AP
429 MtrrSetAllMtrrs (&CpuMpData
->MtrrTable
);
433 Find the current Processor number by APIC ID.
435 @param[in] CpuMpData Pointer to PEI CPU MP Data
436 @param[out] ProcessorNumber Return the pocessor number found
438 @retval EFI_SUCCESS ProcessorNumber is found and returned.
439 @retval EFI_NOT_FOUND ProcessorNumber is not found.
443 IN CPU_MP_DATA
*CpuMpData
,
444 OUT UINTN
*ProcessorNumber
447 UINTN TotalProcessorNumber
;
449 CPU_INFO_IN_HOB
*CpuInfoInHob
;
450 UINT32 CurrentApicId
;
452 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
454 TotalProcessorNumber
= CpuMpData
->CpuCount
;
455 CurrentApicId
= GetApicId ();
456 for (Index
= 0; Index
< TotalProcessorNumber
; Index
++) {
457 if (CpuInfoInHob
[Index
].ApicId
== CurrentApicId
) {
458 *ProcessorNumber
= Index
;
463 return EFI_NOT_FOUND
;
467 This function will get CPU count in the system.
469 @param[in] CpuMpData Pointer to PEI CPU MP Data
471 @return CPU count detected
474 CollectProcessorCount (
475 IN CPU_MP_DATA
*CpuMpData
479 CPU_INFO_IN_HOB
*CpuInfoInHob
;
483 // Send 1st broadcast IPI to APs to wakeup APs
485 CpuMpData
->InitFlag
= ApInitConfig
;
486 WakeUpAP (CpuMpData
, TRUE
, 0, NULL
, NULL
, TRUE
);
487 CpuMpData
->InitFlag
= ApInitDone
;
488 ASSERT (CpuMpData
->CpuCount
<= PcdGet32 (PcdCpuMaxLogicalProcessorNumber
));
490 // Wait for all APs finished the initialization
492 while (CpuMpData
->FinishedCount
< (CpuMpData
->CpuCount
- 1)) {
498 // Enable x2APIC mode if
499 // 1. Number of CPU is greater than 255; or
500 // 2. There are any logical processors reporting an Initial APIC ID of 255 or greater.
503 if (CpuMpData
->CpuCount
> 255) {
505 // If there are more than 255 processor found, force to enable X2APIC
509 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
510 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
511 if (CpuInfoInHob
[Index
].InitialApicId
>= 0xFF) {
519 DEBUG ((DEBUG_INFO
, "Force x2APIC mode!\n"));
521 // Wakeup all APs to enable x2APIC mode
523 WakeUpAP (CpuMpData
, TRUE
, 0, ApFuncEnableX2Apic
, NULL
, TRUE
);
525 // Wait for all known APs finished
527 while (CpuMpData
->FinishedCount
< (CpuMpData
->CpuCount
- 1)) {
531 // Enable x2APIC on BSP
533 SetApicMode (LOCAL_APIC_MODE_X2APIC
);
535 // Set BSP/Aps state to IDLE
537 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
538 SetApState (&CpuMpData
->CpuData
[Index
], CpuStateIdle
);
541 DEBUG ((DEBUG_INFO
, "APIC MODE is %d\n", GetApicMode ()));
543 // Sort BSP/Aps by CPU APIC ID in ascending order
545 SortApicId (CpuMpData
);
547 DEBUG ((DEBUG_INFO
, "MpInitLib: Find %d processors in system.\n", CpuMpData
->CpuCount
));
549 return CpuMpData
->CpuCount
;
553 Initialize CPU AP Data when AP is wakeup at the first time.
555 @param[in, out] CpuMpData Pointer to PEI CPU MP Data
556 @param[in] ProcessorNumber The handle number of processor
557 @param[in] BistData Processor BIST data
558 @param[in] ApTopOfStack Top of AP stack
563 IN OUT CPU_MP_DATA
*CpuMpData
,
564 IN UINTN ProcessorNumber
,
566 IN UINT64 ApTopOfStack
569 CPU_INFO_IN_HOB
*CpuInfoInHob
;
570 MSR_IA32_PLATFORM_ID_REGISTER PlatformIdMsr
;
572 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
573 CpuInfoInHob
[ProcessorNumber
].InitialApicId
= GetInitialApicId ();
574 CpuInfoInHob
[ProcessorNumber
].ApicId
= GetApicId ();
575 CpuInfoInHob
[ProcessorNumber
].Health
= BistData
;
576 CpuInfoInHob
[ProcessorNumber
].ApTopOfStack
= ApTopOfStack
;
578 CpuMpData
->CpuData
[ProcessorNumber
].Waiting
= FALSE
;
579 CpuMpData
->CpuData
[ProcessorNumber
].CpuHealthy
= (BistData
== 0) ? TRUE
: FALSE
;
582 // NOTE: PlatformId is not relevant on AMD platforms.
584 if (!StandardSignatureIsAuthenticAMD ()) {
585 PlatformIdMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_PLATFORM_ID
);
586 CpuMpData
->CpuData
[ProcessorNumber
].PlatformId
= (UINT8
)PlatformIdMsr
.Bits
.PlatformId
;
591 &CpuMpData
->CpuData
[ProcessorNumber
].ProcessorSignature
,
597 InitializeSpinLock(&CpuMpData
->CpuData
[ProcessorNumber
].ApLock
);
598 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateIdle
);
602 Get Protected mode code segment with 16-bit default addressing
603 from current GDT table.
605 @return Protected mode 16-bit code segment value.
609 GetProtectedMode16CS (
613 IA32_DESCRIPTOR GdtrDesc
;
614 IA32_SEGMENT_DESCRIPTOR
*GdtEntry
;
619 AsmReadGdtr (&GdtrDesc
);
620 GdtEntryCount
= (GdtrDesc
.Limit
+ 1) / sizeof (IA32_SEGMENT_DESCRIPTOR
);
621 GdtEntry
= (IA32_SEGMENT_DESCRIPTOR
*) GdtrDesc
.Base
;
622 for (Index
= 0; Index
< GdtEntryCount
; Index
++) {
623 if (GdtEntry
->Bits
.L
== 0 &&
624 GdtEntry
->Bits
.DB
== 0 &&
625 GdtEntry
->Bits
.Type
> 8) {
630 ASSERT (Index
!= GdtEntryCount
);
635 Get Protected mode code segment with 32-bit default addressing
636 from current GDT table.
638 @return Protected mode 32-bit code segment value.
642 GetProtectedMode32CS (
646 IA32_DESCRIPTOR GdtrDesc
;
647 IA32_SEGMENT_DESCRIPTOR
*GdtEntry
;
652 AsmReadGdtr (&GdtrDesc
);
653 GdtEntryCount
= (GdtrDesc
.Limit
+ 1) / sizeof (IA32_SEGMENT_DESCRIPTOR
);
654 GdtEntry
= (IA32_SEGMENT_DESCRIPTOR
*) GdtrDesc
.Base
;
655 for (Index
= 0; Index
< GdtEntryCount
; Index
++) {
656 if (GdtEntry
->Bits
.L
== 0 &&
657 GdtEntry
->Bits
.DB
== 1 &&
658 GdtEntry
->Bits
.Type
> 8) {
663 ASSERT (Index
!= GdtEntryCount
);
668 Reset an AP when in SEV-ES mode.
670 If successful, this function never returns.
672 @param[in] Ghcb Pointer to the GHCB
673 @param[in] CpuMpData Pointer to CPU MP Data
678 MpInitLibSevEsAPReset (
680 IN CPU_MP_DATA
*CpuMpData
683 UINT16 Code16
, Code32
;
688 Code16
= GetProtectedMode16CS ();
689 Code32
= GetProtectedMode32CS ();
691 if (CpuMpData
->WakeupBufferHigh
!= 0) {
692 APResetFn
= (AP_RESET
*) (CpuMpData
->WakeupBufferHigh
+ CpuMpData
->AddressMap
.SwitchToRealNoNxOffset
);
694 APResetFn
= (AP_RESET
*) (CpuMpData
->MpCpuExchangeInfo
->BufferStart
+ CpuMpData
->AddressMap
.SwitchToRealOffset
);
697 BufferStart
= CpuMpData
->MpCpuExchangeInfo
->BufferStart
;
698 StackStart
= CpuMpData
->SevEsAPResetStackStart
-
699 (AP_RESET_STACK_SIZE
* GetApicId ());
702 // This call never returns.
704 APResetFn (BufferStart
, Code16
, Code32
, StackStart
);
708 This function will be called from AP reset code if BSP uses WakeUpAP.
710 @param[in] ExchangeInfo Pointer to the MP exchange info buffer
711 @param[in] ApIndex Number of current executing AP
716 IN MP_CPU_EXCHANGE_INFO
*ExchangeInfo
,
720 CPU_MP_DATA
*CpuMpData
;
721 UINTN ProcessorNumber
;
722 EFI_AP_PROCEDURE Procedure
;
725 volatile UINT32
*ApStartupSignalBuffer
;
726 CPU_INFO_IN_HOB
*CpuInfoInHob
;
728 UINTN CurrentApicMode
;
731 // AP finished assembly code and begin to execute C code
733 CpuMpData
= ExchangeInfo
->CpuMpData
;
736 // AP's local APIC settings will be lost after received INIT IPI
737 // We need to re-initialize them at here
739 ProgramVirtualWireMode ();
741 // Mask the LINT0 and LINT1 so that AP doesn't enter the system timer interrupt handler.
743 DisableLvtInterrupts ();
744 SyncLocalApicTimerSetting (CpuMpData
);
746 CurrentApicMode
= GetApicMode ();
748 if (CpuMpData
->InitFlag
== ApInitConfig
) {
752 InterlockedIncrement ((UINT32
*) &CpuMpData
->CpuCount
);
753 ProcessorNumber
= ApIndex
;
755 // This is first time AP wakeup, get BIST information from AP stack
757 ApTopOfStack
= CpuMpData
->Buffer
+ (ProcessorNumber
+ 1) * CpuMpData
->CpuApStackSize
;
758 BistData
= *(UINT32
*) ((UINTN
) ApTopOfStack
- sizeof (UINTN
));
760 // CpuMpData->CpuData[0].VolatileRegisters is initialized based on BSP environment,
761 // to initialize AP in InitConfig path.
762 // NOTE: IDTR.BASE stored in CpuMpData->CpuData[0].VolatileRegisters points to a different IDT shared by all APs.
764 RestoreVolatileRegisters (&CpuMpData
->CpuData
[0].VolatileRegisters
, FALSE
);
765 InitializeApData (CpuMpData
, ProcessorNumber
, BistData
, ApTopOfStack
);
766 ApStartupSignalBuffer
= CpuMpData
->CpuData
[ProcessorNumber
].StartupApSignal
;
769 // Delay decrementing the APs executing count when SEV-ES is enabled
770 // to allow the APs to issue an AP_RESET_HOLD before the BSP possibly
771 // performs another INIT-SIPI-SIPI sequence.
773 if (!CpuMpData
->SevEsIsEnabled
) {
774 InterlockedDecrement ((UINT32
*) &CpuMpData
->MpCpuExchangeInfo
->NumApsExecuting
);
778 // Execute AP function if AP is ready
780 GetProcessorNumber (CpuMpData
, &ProcessorNumber
);
782 // Clear AP start-up signal when AP waken up
784 ApStartupSignalBuffer
= CpuMpData
->CpuData
[ProcessorNumber
].StartupApSignal
;
785 InterlockedCompareExchange32 (
786 (UINT32
*) ApStartupSignalBuffer
,
791 if (CpuMpData
->InitFlag
== ApInitReconfig
) {
793 // ApInitReconfig happens when:
794 // 1. AP is re-enabled after it's disabled, in either PEI or DXE phase.
795 // 2. AP is initialized in DXE phase.
796 // In either case, use the volatile registers value derived from BSP.
797 // NOTE: IDTR.BASE stored in CpuMpData->CpuData[0].VolatileRegisters points to a
798 // different IDT shared by all APs.
800 RestoreVolatileRegisters (&CpuMpData
->CpuData
[0].VolatileRegisters
, FALSE
);
802 if (CpuMpData
->ApLoopMode
== ApInHltLoop
) {
804 // Restore AP's volatile registers saved before AP is halted
806 RestoreVolatileRegisters (&CpuMpData
->CpuData
[ProcessorNumber
].VolatileRegisters
, TRUE
);
809 // The CPU driver might not flush TLB for APs on spot after updating
810 // page attributes. AP in mwait loop mode needs to take care of it when
817 if (GetApState (&CpuMpData
->CpuData
[ProcessorNumber
]) == CpuStateReady
) {
818 Procedure
= (EFI_AP_PROCEDURE
)CpuMpData
->CpuData
[ProcessorNumber
].ApFunction
;
819 Parameter
= (VOID
*) CpuMpData
->CpuData
[ProcessorNumber
].ApFunctionArgument
;
820 if (Procedure
!= NULL
) {
821 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateBusy
);
823 // Enable source debugging on AP function
827 // Invoke AP function here
829 Procedure (Parameter
);
830 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
831 if (CpuMpData
->SwitchBspFlag
) {
833 // Re-get the processor number due to BSP/AP maybe exchange in AP function
835 GetProcessorNumber (CpuMpData
, &ProcessorNumber
);
836 CpuMpData
->CpuData
[ProcessorNumber
].ApFunction
= 0;
837 CpuMpData
->CpuData
[ProcessorNumber
].ApFunctionArgument
= 0;
838 ApStartupSignalBuffer
= CpuMpData
->CpuData
[ProcessorNumber
].StartupApSignal
;
839 CpuInfoInHob
[ProcessorNumber
].ApTopOfStack
= CpuInfoInHob
[CpuMpData
->NewBspNumber
].ApTopOfStack
;
841 if (CpuInfoInHob
[ProcessorNumber
].ApicId
!= GetApicId () ||
842 CpuInfoInHob
[ProcessorNumber
].InitialApicId
!= GetInitialApicId ()) {
843 if (CurrentApicMode
!= GetApicMode ()) {
845 // If APIC mode change happened during AP function execution,
846 // we do not support APIC ID value changed.
852 // Re-get the CPU APICID and Initial APICID if they are changed
854 CpuInfoInHob
[ProcessorNumber
].ApicId
= GetApicId ();
855 CpuInfoInHob
[ProcessorNumber
].InitialApicId
= GetInitialApicId ();
860 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateFinished
);
865 // AP finished executing C code
867 InterlockedIncrement ((UINT32
*) &CpuMpData
->FinishedCount
);
870 // Place AP is specified loop mode
872 if (CpuMpData
->ApLoopMode
== ApInHltLoop
) {
874 // Save AP volatile registers
876 SaveVolatileRegisters (&CpuMpData
->CpuData
[ProcessorNumber
].VolatileRegisters
);
878 // Place AP in HLT-loop
881 DisableInterrupts ();
882 if (CpuMpData
->SevEsIsEnabled
) {
883 MSR_SEV_ES_GHCB_REGISTER Msr
;
888 if (CpuMpData
->InitFlag
== ApInitConfig
) {
893 Msr
.GhcbPhysicalAddress
= AsmReadMsr64 (MSR_SEV_ES_GHCB
);
902 // Perform the delayed decrement just before issuing the first
903 // VMGEXIT with AP_RESET_HOLD.
905 InterlockedDecrement ((UINT32
*) &CpuMpData
->MpCpuExchangeInfo
->NumApsExecuting
);
908 Status
= VmgExit (Ghcb
, SVM_EXIT_AP_RESET_HOLD
, 0, 0);
909 if ((Status
== 0) && (Ghcb
->SaveArea
.SwExitInfo2
!= 0)) {
918 // Awakened in a new phase? Use the new CpuMpData
920 if (CpuMpData
->NewCpuMpData
!= NULL
) {
921 CpuMpData
= CpuMpData
->NewCpuMpData
;
924 MpInitLibSevEsAPReset (Ghcb
, CpuMpData
);
932 DisableInterrupts ();
933 if (CpuMpData
->ApLoopMode
== ApInMwaitLoop
) {
935 // Place AP in MWAIT-loop
937 AsmMonitor ((UINTN
) ApStartupSignalBuffer
, 0, 0);
938 if (*ApStartupSignalBuffer
!= WAKEUP_AP_SIGNAL
) {
940 // Check AP start-up signal again.
941 // If AP start-up signal is not set, place AP into
942 // the specified C-state
944 AsmMwait (CpuMpData
->ApTargetCState
<< 4, 0);
946 } else if (CpuMpData
->ApLoopMode
== ApInRunLoop
) {
948 // Place AP in Run-loop
956 // If AP start-up signal is written, AP is waken up
957 // otherwise place AP in loop again
959 if (*ApStartupSignalBuffer
== WAKEUP_AP_SIGNAL
) {
967 Wait for AP wakeup and write AP start-up signal till AP is waken up.
969 @param[in] ApStartupSignalBuffer Pointer to AP wakeup signal
973 IN
volatile UINT32
*ApStartupSignalBuffer
977 // If AP is waken up, StartupApSignal should be cleared.
978 // Otherwise, write StartupApSignal again till AP waken up.
980 while (InterlockedCompareExchange32 (
981 (UINT32
*) ApStartupSignalBuffer
,
990 This function will fill the exchange info structure.
992 @param[in] CpuMpData Pointer to CPU MP Data
996 FillExchangeInfoData (
997 IN CPU_MP_DATA
*CpuMpData
1000 volatile MP_CPU_EXCHANGE_INFO
*ExchangeInfo
;
1002 IA32_SEGMENT_DESCRIPTOR
*Selector
;
1005 ExchangeInfo
= CpuMpData
->MpCpuExchangeInfo
;
1006 ExchangeInfo
->Lock
= 0;
1007 ExchangeInfo
->StackStart
= CpuMpData
->Buffer
;
1008 ExchangeInfo
->StackSize
= CpuMpData
->CpuApStackSize
;
1009 ExchangeInfo
->BufferStart
= CpuMpData
->WakeupBuffer
;
1010 ExchangeInfo
->ModeOffset
= CpuMpData
->AddressMap
.ModeEntryOffset
;
1012 ExchangeInfo
->CodeSegment
= AsmReadCs ();
1013 ExchangeInfo
->DataSegment
= AsmReadDs ();
1015 ExchangeInfo
->Cr3
= AsmReadCr3 ();
1017 ExchangeInfo
->CFunction
= (UINTN
) ApWakeupFunction
;
1018 ExchangeInfo
->ApIndex
= 0;
1019 ExchangeInfo
->NumApsExecuting
= 0;
1020 ExchangeInfo
->InitFlag
= (UINTN
) CpuMpData
->InitFlag
;
1021 ExchangeInfo
->CpuInfo
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
1022 ExchangeInfo
->CpuMpData
= CpuMpData
;
1024 ExchangeInfo
->EnableExecuteDisable
= IsBspExecuteDisableEnabled ();
1026 ExchangeInfo
->InitializeFloatingPointUnitsAddress
= (UINTN
)InitializeFloatingPointUnits
;
1029 // We can check either CPUID(7).ECX[bit16] or check CR4.LA57[bit12]
1030 // to determin whether 5-Level Paging is enabled.
1031 // CPUID(7).ECX[bit16] shows CPU's capability, CR4.LA57[bit12] shows
1032 // current system setting.
1033 // Using latter way is simpler because it also eliminates the needs to
1034 // check whether platform wants to enable it.
1036 Cr4
.UintN
= AsmReadCr4 ();
1037 ExchangeInfo
->Enable5LevelPaging
= (BOOLEAN
) (Cr4
.Bits
.LA57
== 1);
1038 DEBUG ((DEBUG_INFO
, "%a: 5-Level Paging = %d\n", gEfiCallerBaseName
, ExchangeInfo
->Enable5LevelPaging
));
1040 ExchangeInfo
->SevEsIsEnabled
= CpuMpData
->SevEsIsEnabled
;
1041 ExchangeInfo
->GhcbBase
= (UINTN
) CpuMpData
->GhcbBase
;
1044 // Get the BSP's data of GDT and IDT
1046 AsmReadGdtr ((IA32_DESCRIPTOR
*) &ExchangeInfo
->GdtrProfile
);
1047 AsmReadIdtr ((IA32_DESCRIPTOR
*) &ExchangeInfo
->IdtrProfile
);
1050 // Find a 32-bit code segment
1052 Selector
= (IA32_SEGMENT_DESCRIPTOR
*)ExchangeInfo
->GdtrProfile
.Base
;
1053 Size
= ExchangeInfo
->GdtrProfile
.Limit
+ 1;
1055 if (Selector
->Bits
.L
== 0 && Selector
->Bits
.Type
>= 8) {
1056 ExchangeInfo
->ModeTransitionSegment
=
1057 (UINT16
)((UINTN
)Selector
- ExchangeInfo
->GdtrProfile
.Base
);
1061 Size
-= sizeof (IA32_SEGMENT_DESCRIPTOR
);
1065 // Copy all 32-bit code and 64-bit code into memory with type of
1066 // EfiBootServicesCode to avoid page fault if NX memory protection is enabled.
1068 if (CpuMpData
->WakeupBufferHigh
!= 0) {
1069 Size
= CpuMpData
->AddressMap
.RendezvousFunnelSize
+
1070 CpuMpData
->AddressMap
.SwitchToRealSize
-
1071 CpuMpData
->AddressMap
.ModeTransitionOffset
;
1073 (VOID
*)CpuMpData
->WakeupBufferHigh
,
1074 CpuMpData
->AddressMap
.RendezvousFunnelAddress
+
1075 CpuMpData
->AddressMap
.ModeTransitionOffset
,
1079 ExchangeInfo
->ModeTransitionMemory
= (UINT32
)CpuMpData
->WakeupBufferHigh
;
1081 ExchangeInfo
->ModeTransitionMemory
= (UINT32
)
1082 (ExchangeInfo
->BufferStart
+ CpuMpData
->AddressMap
.ModeTransitionOffset
);
1085 ExchangeInfo
->ModeHighMemory
= ExchangeInfo
->ModeTransitionMemory
+
1086 (UINT32
)ExchangeInfo
->ModeOffset
-
1087 (UINT32
)CpuMpData
->AddressMap
.ModeTransitionOffset
;
1088 ExchangeInfo
->ModeHighSegment
= (UINT16
)ExchangeInfo
->CodeSegment
;
1092 Helper function that waits until the finished AP count reaches the specified
1093 limit, or the specified timeout elapses (whichever comes first).
1095 @param[in] CpuMpData Pointer to CPU MP Data.
1096 @param[in] FinishedApLimit The number of finished APs to wait for.
1097 @param[in] TimeLimit The number of microseconds to wait for.
1100 TimedWaitForApFinish (
1101 IN CPU_MP_DATA
*CpuMpData
,
1102 IN UINT32 FinishedApLimit
,
1107 Get available system memory below 1MB by specified size.
1109 @param[in] CpuMpData The pointer to CPU MP Data structure.
1112 BackupAndPrepareWakeupBuffer(
1113 IN CPU_MP_DATA
*CpuMpData
1117 (VOID
*) CpuMpData
->BackupBuffer
,
1118 (VOID
*) CpuMpData
->WakeupBuffer
,
1119 CpuMpData
->BackupBufferSize
1122 (VOID
*) CpuMpData
->WakeupBuffer
,
1123 (VOID
*) CpuMpData
->AddressMap
.RendezvousFunnelAddress
,
1124 CpuMpData
->AddressMap
.RendezvousFunnelSize
+
1125 CpuMpData
->AddressMap
.SwitchToRealSize
1130 Restore wakeup buffer data.
1132 @param[in] CpuMpData The pointer to CPU MP Data structure.
1135 RestoreWakeupBuffer(
1136 IN CPU_MP_DATA
*CpuMpData
1140 (VOID
*) CpuMpData
->WakeupBuffer
,
1141 (VOID
*) CpuMpData
->BackupBuffer
,
1142 CpuMpData
->BackupBufferSize
1147 Calculate the size of the reset stack.
1149 @return Total amount of memory required for stacks
1153 GetApResetStackSize (
1157 return AP_RESET_STACK_SIZE
* PcdGet32(PcdCpuMaxLogicalProcessorNumber
);
1161 Calculate the size of the reset vector.
1163 @param[in] AddressMap The pointer to Address Map structure.
1165 @return Total amount of memory required for the AP reset area
1169 GetApResetVectorSize (
1170 IN MP_ASSEMBLY_ADDRESS_MAP
*AddressMap
1175 Size
= ALIGN_VALUE (AddressMap
->RendezvousFunnelSize
+
1176 AddressMap
->SwitchToRealSize
+
1177 sizeof (MP_CPU_EXCHANGE_INFO
),
1178 CPU_STACK_ALIGNMENT
);
1179 Size
+= GetApResetStackSize ();
1185 Allocate reset vector buffer.
1187 @param[in, out] CpuMpData The pointer to CPU MP Data structure.
1190 AllocateResetVector (
1191 IN OUT CPU_MP_DATA
*CpuMpData
1194 UINTN ApResetVectorSize
;
1196 if (CpuMpData
->WakeupBuffer
== (UINTN
) -1) {
1197 ApResetVectorSize
= GetApResetVectorSize (&CpuMpData
->AddressMap
);
1199 CpuMpData
->WakeupBuffer
= GetWakeupBuffer (ApResetVectorSize
);
1200 CpuMpData
->MpCpuExchangeInfo
= (MP_CPU_EXCHANGE_INFO
*) (UINTN
)
1201 (CpuMpData
->WakeupBuffer
+
1202 CpuMpData
->AddressMap
.RendezvousFunnelSize
+
1203 CpuMpData
->AddressMap
.SwitchToRealSize
);
1204 CpuMpData
->WakeupBufferHigh
= GetModeTransitionBuffer (
1205 CpuMpData
->AddressMap
.RendezvousFunnelSize
+
1206 CpuMpData
->AddressMap
.SwitchToRealSize
-
1207 CpuMpData
->AddressMap
.ModeTransitionOffset
1210 // The reset stack starts at the end of the buffer.
1212 CpuMpData
->SevEsAPResetStackStart
= CpuMpData
->WakeupBuffer
+ ApResetVectorSize
;
1214 BackupAndPrepareWakeupBuffer (CpuMpData
);
1218 Free AP reset vector buffer.
1220 @param[in] CpuMpData The pointer to CPU MP Data structure.
1224 IN CPU_MP_DATA
*CpuMpData
1228 // If SEV-ES is enabled, the reset area is needed for AP parking and
1229 // and AP startup in the OS, so the reset area is reserved. Do not
1230 // perform the restore as this will overwrite memory which has data
1231 // needed by SEV-ES.
1233 if (!CpuMpData
->SevEsIsEnabled
) {
1234 RestoreWakeupBuffer (CpuMpData
);
1239 Allocate the SEV-ES AP jump table buffer.
1241 @param[in, out] CpuMpData The pointer to CPU MP Data structure.
1244 AllocateSevEsAPMemory (
1245 IN OUT CPU_MP_DATA
*CpuMpData
1248 if (CpuMpData
->SevEsAPBuffer
== (UINTN
) -1) {
1249 CpuMpData
->SevEsAPBuffer
=
1250 CpuMpData
->SevEsIsEnabled
? GetSevEsAPMemory () : 0;
1255 Program the SEV-ES AP jump table buffer.
1257 @param[in] SipiVector The SIPI vector used for the AP Reset
1264 SEV_ES_AP_JMP_FAR
*JmpFar
;
1265 UINT32 Offset
, InsnByte
;
1268 JmpFar
= (SEV_ES_AP_JMP_FAR
*) FixedPcdGet32 (PcdSevEsWorkAreaBase
);
1269 ASSERT (JmpFar
!= NULL
);
1272 // Obtain the address of the Segment/Rip location in the workarea.
1273 // This will be set to a value derived from the SIPI vector and will
1274 // be the memory address used for the far jump below.
1276 Offset
= FixedPcdGet32 (PcdSevEsWorkAreaBase
);
1277 Offset
+= sizeof (JmpFar
->InsnBuffer
);
1278 LoNib
= (UINT8
) Offset
;
1279 HiNib
= (UINT8
) (Offset
>> 8);
1282 // Program the workarea (which is the initial AP boot address) with
1283 // far jump to the SIPI vector (where XX and YY represent the
1284 // address of where the SIPI vector is stored.
1286 // JMP FAR [CS:XXYY] => 2E FF 2E YY XX
1289 JmpFar
->InsnBuffer
[InsnByte
++] = 0x2E; // CS override prefix
1290 JmpFar
->InsnBuffer
[InsnByte
++] = 0xFF; // JMP (FAR)
1291 JmpFar
->InsnBuffer
[InsnByte
++] = 0x2E; // ModRM (JMP memory location)
1292 JmpFar
->InsnBuffer
[InsnByte
++] = LoNib
; // YY offset ...
1293 JmpFar
->InsnBuffer
[InsnByte
++] = HiNib
; // XX offset ...
1296 // Program the Segment/Rip based on the SIPI vector (always at least
1297 // 16-byte aligned, so Rip is set to 0).
1300 JmpFar
->Segment
= (UINT16
) (SipiVector
>> 4);
1304 This function will be called by BSP to wakeup AP.
1306 @param[in] CpuMpData Pointer to CPU MP Data
1307 @param[in] Broadcast TRUE: Send broadcast IPI to all APs
1308 FALSE: Send IPI to AP by ApicId
1309 @param[in] ProcessorNumber The handle number of specified processor
1310 @param[in] Procedure The function to be invoked by AP
1311 @param[in] ProcedureArgument The argument to be passed into AP function
1312 @param[in] WakeUpDisabledAps Whether need to wake up disabled APs in broadcast mode.
1316 IN CPU_MP_DATA
*CpuMpData
,
1317 IN BOOLEAN Broadcast
,
1318 IN UINTN ProcessorNumber
,
1319 IN EFI_AP_PROCEDURE Procedure
, OPTIONAL
1320 IN VOID
*ProcedureArgument
, OPTIONAL
1321 IN BOOLEAN WakeUpDisabledAps
1324 volatile MP_CPU_EXCHANGE_INFO
*ExchangeInfo
;
1326 CPU_AP_DATA
*CpuData
;
1327 BOOLEAN ResetVectorRequired
;
1328 CPU_INFO_IN_HOB
*CpuInfoInHob
;
1330 CpuMpData
->FinishedCount
= 0;
1331 ResetVectorRequired
= FALSE
;
1333 if (CpuMpData
->WakeUpByInitSipiSipi
||
1334 CpuMpData
->InitFlag
!= ApInitDone
) {
1335 ResetVectorRequired
= TRUE
;
1336 AllocateResetVector (CpuMpData
);
1337 AllocateSevEsAPMemory (CpuMpData
);
1338 FillExchangeInfoData (CpuMpData
);
1339 SaveLocalApicTimerSetting (CpuMpData
);
1342 if (CpuMpData
->ApLoopMode
== ApInMwaitLoop
) {
1344 // Get AP target C-state each time when waking up AP,
1345 // for it maybe updated by platform again
1347 CpuMpData
->ApTargetCState
= PcdGet8 (PcdCpuApTargetCstate
);
1350 ExchangeInfo
= CpuMpData
->MpCpuExchangeInfo
;
1353 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
1354 if (Index
!= CpuMpData
->BspNumber
) {
1355 CpuData
= &CpuMpData
->CpuData
[Index
];
1357 // All AP(include disabled AP) will be woke up by INIT-SIPI-SIPI, but
1358 // the AP procedure will be skipped for disabled AP because AP state
1359 // is not CpuStateReady.
1361 if (GetApState (CpuData
) == CpuStateDisabled
&& !WakeUpDisabledAps
) {
1365 CpuData
->ApFunction
= (UINTN
) Procedure
;
1366 CpuData
->ApFunctionArgument
= (UINTN
) ProcedureArgument
;
1367 SetApState (CpuData
, CpuStateReady
);
1368 if (CpuMpData
->InitFlag
!= ApInitConfig
) {
1369 *(UINT32
*) CpuData
->StartupApSignal
= WAKEUP_AP_SIGNAL
;
1373 if (ResetVectorRequired
) {
1375 // For SEV-ES, the initial AP boot address will be defined by
1376 // PcdSevEsWorkAreaBase. The Segment/Rip must be the jump address
1377 // from the original INIT-SIPI-SIPI.
1379 if (CpuMpData
->SevEsIsEnabled
) {
1380 SetSevEsJumpTable (ExchangeInfo
->BufferStart
);
1386 SendInitSipiSipiAllExcludingSelf ((UINT32
) ExchangeInfo
->BufferStart
);
1388 if (CpuMpData
->InitFlag
== ApInitConfig
) {
1389 if (PcdGet32 (PcdCpuBootLogicalProcessorNumber
) > 0) {
1391 // The AP enumeration algorithm below is suitable only when the
1392 // platform can tell us the *exact* boot CPU count in advance.
1394 // The wait below finishes only when the detected AP count reaches
1395 // (PcdCpuBootLogicalProcessorNumber - 1), regardless of how long that
1396 // takes. If at least one AP fails to check in (meaning a platform
1397 // hardware bug), the detection hangs forever, by design. If the actual
1398 // boot CPU count in the system is higher than
1399 // PcdCpuBootLogicalProcessorNumber (meaning a platform
1400 // misconfiguration), then some APs may complete initialization after
1401 // the wait finishes, and cause undefined behavior.
1403 TimedWaitForApFinish (
1405 PcdGet32 (PcdCpuBootLogicalProcessorNumber
) - 1,
1406 MAX_UINT32
// approx. 71 minutes
1410 // The AP enumeration algorithm below is suitable for two use cases.
1412 // (1) The check-in time for an individual AP is bounded, and APs run
1413 // through their initialization routines strongly concurrently. In
1414 // particular, the number of concurrently running APs
1415 // ("NumApsExecuting") is never expected to fall to zero
1416 // *temporarily* -- it is expected to fall to zero only when all
1417 // APs have checked-in.
1419 // In this case, the platform is supposed to set
1420 // PcdCpuApInitTimeOutInMicroSeconds to a low-ish value (just long
1421 // enough for one AP to start initialization). The timeout will be
1422 // reached soon, and remaining APs are collected by watching
1423 // NumApsExecuting fall to zero. If NumApsExecuting falls to zero
1424 // mid-process, while some APs have not completed initialization,
1425 // the behavior is undefined.
1427 // (2) The check-in time for an individual AP is unbounded, and/or APs
1428 // may complete their initializations widely spread out. In
1429 // particular, some APs may finish initialization before some APs
1432 // In this case, the platform is supposed to set
1433 // PcdCpuApInitTimeOutInMicroSeconds to a high-ish value. The AP
1434 // enumeration will always take that long (except when the boot CPU
1435 // count happens to be maximal, that is,
1436 // PcdCpuMaxLogicalProcessorNumber). All APs are expected to
1437 // check-in before the timeout, and NumApsExecuting is assumed zero
1438 // at timeout. APs that miss the time-out may cause undefined
1441 TimedWaitForApFinish (
1443 PcdGet32 (PcdCpuMaxLogicalProcessorNumber
) - 1,
1444 PcdGet32 (PcdCpuApInitTimeOutInMicroSeconds
)
1447 while (CpuMpData
->MpCpuExchangeInfo
->NumApsExecuting
!= 0) {
1453 // Wait all APs waken up if this is not the 1st broadcast of SIPI
1455 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
1456 CpuData
= &CpuMpData
->CpuData
[Index
];
1457 if (Index
!= CpuMpData
->BspNumber
) {
1458 WaitApWakeup (CpuData
->StartupApSignal
);
1463 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
1464 CpuData
->ApFunction
= (UINTN
) Procedure
;
1465 CpuData
->ApFunctionArgument
= (UINTN
) ProcedureArgument
;
1466 SetApState (CpuData
, CpuStateReady
);
1468 // Wakeup specified AP
1470 ASSERT (CpuMpData
->InitFlag
!= ApInitConfig
);
1471 *(UINT32
*) CpuData
->StartupApSignal
= WAKEUP_AP_SIGNAL
;
1472 if (ResetVectorRequired
) {
1473 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
1476 // For SEV-ES, the initial AP boot address will be defined by
1477 // PcdSevEsWorkAreaBase. The Segment/Rip must be the jump address
1478 // from the original INIT-SIPI-SIPI.
1480 if (CpuMpData
->SevEsIsEnabled
) {
1481 SetSevEsJumpTable (ExchangeInfo
->BufferStart
);
1485 CpuInfoInHob
[ProcessorNumber
].ApicId
,
1486 (UINT32
) ExchangeInfo
->BufferStart
1490 // Wait specified AP waken up
1492 WaitApWakeup (CpuData
->StartupApSignal
);
1495 if (ResetVectorRequired
) {
1496 FreeResetVector (CpuMpData
);
1500 // After one round of Wakeup Ap actions, need to re-sync ApLoopMode with
1501 // WakeUpByInitSipiSipi flag. WakeUpByInitSipiSipi flag maybe changed by
1502 // S3SmmInitDone Ppi.
1504 CpuMpData
->WakeUpByInitSipiSipi
= (CpuMpData
->ApLoopMode
== ApInHltLoop
);
1508 Calculate timeout value and return the current performance counter value.
1510 Calculate the number of performance counter ticks required for a timeout.
1511 If TimeoutInMicroseconds is 0, return value is also 0, which is recognized
1514 @param[in] TimeoutInMicroseconds Timeout value in microseconds.
1515 @param[out] CurrentTime Returns the current value of the performance counter.
1517 @return Expected time stamp counter for timeout.
1518 If TimeoutInMicroseconds is 0, return value is also 0, which is recognized
1524 IN UINTN TimeoutInMicroseconds
,
1525 OUT UINT64
*CurrentTime
1528 UINT64 TimeoutInSeconds
;
1529 UINT64 TimestampCounterFreq
;
1532 // Read the current value of the performance counter
1534 *CurrentTime
= GetPerformanceCounter ();
1537 // If TimeoutInMicroseconds is 0, return value is also 0, which is recognized
1540 if (TimeoutInMicroseconds
== 0) {
1545 // GetPerformanceCounterProperties () returns the timestamp counter's frequency
1548 TimestampCounterFreq
= GetPerformanceCounterProperties (NULL
, NULL
);
1551 // Check the potential overflow before calculate the number of ticks for the timeout value.
1553 if (DivU64x64Remainder (MAX_UINT64
, TimeoutInMicroseconds
, NULL
) < TimestampCounterFreq
) {
1555 // Convert microseconds into seconds if direct multiplication overflows
1557 TimeoutInSeconds
= DivU64x32 (TimeoutInMicroseconds
, 1000000);
1559 // Assertion if the final tick count exceeds MAX_UINT64
1561 ASSERT (DivU64x64Remainder (MAX_UINT64
, TimeoutInSeconds
, NULL
) >= TimestampCounterFreq
);
1562 return MultU64x64 (TimestampCounterFreq
, TimeoutInSeconds
);
1565 // No overflow case, multiply the return value with TimeoutInMicroseconds and then divide
1566 // it by 1,000,000, to get the number of ticks for the timeout value.
1570 TimestampCounterFreq
,
1571 TimeoutInMicroseconds
1579 Checks whether timeout expires.
1581 Check whether the number of elapsed performance counter ticks required for
1582 a timeout condition has been reached.
1583 If Timeout is zero, which means infinity, return value is always FALSE.
1585 @param[in, out] PreviousTime On input, the value of the performance counter
1586 when it was last read.
1587 On output, the current value of the performance
1589 @param[in] TotalTime The total amount of elapsed time in performance
1591 @param[in] Timeout The number of performance counter ticks required
1592 to reach a timeout condition.
1594 @retval TRUE A timeout condition has been reached.
1595 @retval FALSE A timeout condition has not been reached.
1600 IN OUT UINT64
*PreviousTime
,
1601 IN UINT64
*TotalTime
,
1614 GetPerformanceCounterProperties (&Start
, &End
);
1615 Cycle
= End
- Start
;
1620 CurrentTime
= GetPerformanceCounter();
1621 Delta
= (INT64
) (CurrentTime
- *PreviousTime
);
1628 *TotalTime
+= Delta
;
1629 *PreviousTime
= CurrentTime
;
1630 if (*TotalTime
> Timeout
) {
1637 Helper function that waits until the finished AP count reaches the specified
1638 limit, or the specified timeout elapses (whichever comes first).
1640 @param[in] CpuMpData Pointer to CPU MP Data.
1641 @param[in] FinishedApLimit The number of finished APs to wait for.
1642 @param[in] TimeLimit The number of microseconds to wait for.
1645 TimedWaitForApFinish (
1646 IN CPU_MP_DATA
*CpuMpData
,
1647 IN UINT32 FinishedApLimit
,
1652 // CalculateTimeout() and CheckTimeout() consider a TimeLimit of 0
1653 // "infinity", so check for (TimeLimit == 0) explicitly.
1655 if (TimeLimit
== 0) {
1659 CpuMpData
->TotalTime
= 0;
1660 CpuMpData
->ExpectedTime
= CalculateTimeout (
1662 &CpuMpData
->CurrentTime
1664 while (CpuMpData
->FinishedCount
< FinishedApLimit
&&
1666 &CpuMpData
->CurrentTime
,
1667 &CpuMpData
->TotalTime
,
1668 CpuMpData
->ExpectedTime
1673 if (CpuMpData
->FinishedCount
>= FinishedApLimit
) {
1676 "%a: reached FinishedApLimit=%u in %Lu microseconds\n",
1679 DivU64x64Remainder (
1680 MultU64x32 (CpuMpData
->TotalTime
, 1000000),
1681 GetPerformanceCounterProperties (NULL
, NULL
),
1689 Reset an AP to Idle state.
1691 Any task being executed by the AP will be aborted and the AP
1692 will be waiting for a new task in Wait-For-SIPI state.
1694 @param[in] ProcessorNumber The handle number of processor.
1697 ResetProcessorToIdleState (
1698 IN UINTN ProcessorNumber
1701 CPU_MP_DATA
*CpuMpData
;
1703 CpuMpData
= GetCpuMpData ();
1705 CpuMpData
->InitFlag
= ApInitReconfig
;
1706 WakeUpAP (CpuMpData
, FALSE
, ProcessorNumber
, NULL
, NULL
, TRUE
);
1707 while (CpuMpData
->FinishedCount
< 1) {
1710 CpuMpData
->InitFlag
= ApInitDone
;
1712 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateIdle
);
1716 Searches for the next waiting AP.
1718 Search for the next AP that is put in waiting state by single-threaded StartupAllAPs().
1720 @param[out] NextProcessorNumber Pointer to the processor number of the next waiting AP.
1722 @retval EFI_SUCCESS The next waiting AP has been found.
1723 @retval EFI_NOT_FOUND No waiting AP exists.
1727 GetNextWaitingProcessorNumber (
1728 OUT UINTN
*NextProcessorNumber
1731 UINTN ProcessorNumber
;
1732 CPU_MP_DATA
*CpuMpData
;
1734 CpuMpData
= GetCpuMpData ();
1736 for (ProcessorNumber
= 0; ProcessorNumber
< CpuMpData
->CpuCount
; ProcessorNumber
++) {
1737 if (CpuMpData
->CpuData
[ProcessorNumber
].Waiting
) {
1738 *NextProcessorNumber
= ProcessorNumber
;
1743 return EFI_NOT_FOUND
;
1746 /** Checks status of specified AP.
1748 This function checks whether the specified AP has finished the task assigned
1749 by StartupThisAP(), and whether timeout expires.
1751 @param[in] ProcessorNumber The handle number of processor.
1753 @retval EFI_SUCCESS Specified AP has finished task assigned by StartupThisAPs().
1754 @retval EFI_TIMEOUT The timeout expires.
1755 @retval EFI_NOT_READY Specified AP has not finished task and timeout has not expired.
1759 IN UINTN ProcessorNumber
1762 CPU_MP_DATA
*CpuMpData
;
1763 CPU_AP_DATA
*CpuData
;
1765 CpuMpData
= GetCpuMpData ();
1766 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
1769 // Check the CPU state of AP. If it is CpuStateIdle, then the AP has finished its task.
1770 // Only BSP and corresponding AP access this unit of CPU Data. This means the AP will not modify the
1771 // value of state after setting the it to CpuStateIdle, so BSP can safely make use of its value.
1774 // If the AP finishes for StartupThisAP(), return EFI_SUCCESS.
1776 if (GetApState(CpuData
) == CpuStateFinished
) {
1777 if (CpuData
->Finished
!= NULL
) {
1778 *(CpuData
->Finished
) = TRUE
;
1780 SetApState (CpuData
, CpuStateIdle
);
1784 // If timeout expires for StartupThisAP(), report timeout.
1786 if (CheckTimeout (&CpuData
->CurrentTime
, &CpuData
->TotalTime
, CpuData
->ExpectedTime
)) {
1787 if (CpuData
->Finished
!= NULL
) {
1788 *(CpuData
->Finished
) = FALSE
;
1791 // Reset failed AP to idle state
1793 ResetProcessorToIdleState (ProcessorNumber
);
1798 return EFI_NOT_READY
;
1802 Checks status of all APs.
1804 This function checks whether all APs have finished task assigned by StartupAllAPs(),
1805 and whether timeout expires.
1807 @retval EFI_SUCCESS All APs have finished task assigned by StartupAllAPs().
1808 @retval EFI_TIMEOUT The timeout expires.
1809 @retval EFI_NOT_READY APs have not finished task and timeout has not expired.
1816 UINTN ProcessorNumber
;
1817 UINTN NextProcessorNumber
;
1820 CPU_MP_DATA
*CpuMpData
;
1821 CPU_AP_DATA
*CpuData
;
1823 CpuMpData
= GetCpuMpData ();
1825 NextProcessorNumber
= 0;
1828 // Go through all APs that are responsible for the StartupAllAPs().
1830 for (ProcessorNumber
= 0; ProcessorNumber
< CpuMpData
->CpuCount
; ProcessorNumber
++) {
1831 if (!CpuMpData
->CpuData
[ProcessorNumber
].Waiting
) {
1835 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
1837 // Check the CPU state of AP. If it is CpuStateIdle, then the AP has finished its task.
1838 // Only BSP and corresponding AP access this unit of CPU Data. This means the AP will not modify the
1839 // value of state after setting the it to CpuStateIdle, so BSP can safely make use of its value.
1841 if (GetApState(CpuData
) == CpuStateFinished
) {
1842 CpuMpData
->RunningCount
--;
1843 CpuMpData
->CpuData
[ProcessorNumber
].Waiting
= FALSE
;
1844 SetApState(CpuData
, CpuStateIdle
);
1847 // If in Single Thread mode, then search for the next waiting AP for execution.
1849 if (CpuMpData
->SingleThread
) {
1850 Status
= GetNextWaitingProcessorNumber (&NextProcessorNumber
);
1852 if (!EFI_ERROR (Status
)) {
1856 (UINT32
) NextProcessorNumber
,
1857 CpuMpData
->Procedure
,
1858 CpuMpData
->ProcArguments
,
1867 // If all APs finish, return EFI_SUCCESS.
1869 if (CpuMpData
->RunningCount
== 0) {
1874 // If timeout expires, report timeout.
1877 &CpuMpData
->CurrentTime
,
1878 &CpuMpData
->TotalTime
,
1879 CpuMpData
->ExpectedTime
)
1882 // If FailedCpuList is not NULL, record all failed APs in it.
1884 if (CpuMpData
->FailedCpuList
!= NULL
) {
1885 *CpuMpData
->FailedCpuList
=
1886 AllocatePool ((CpuMpData
->RunningCount
+ 1) * sizeof (UINTN
));
1887 ASSERT (*CpuMpData
->FailedCpuList
!= NULL
);
1891 for (ProcessorNumber
= 0; ProcessorNumber
< CpuMpData
->CpuCount
; ProcessorNumber
++) {
1893 // Check whether this processor is responsible for StartupAllAPs().
1895 if (CpuMpData
->CpuData
[ProcessorNumber
].Waiting
) {
1897 // Reset failed APs to idle state
1899 ResetProcessorToIdleState (ProcessorNumber
);
1900 CpuMpData
->CpuData
[ProcessorNumber
].Waiting
= FALSE
;
1901 if (CpuMpData
->FailedCpuList
!= NULL
) {
1902 (*CpuMpData
->FailedCpuList
)[ListIndex
++] = ProcessorNumber
;
1906 if (CpuMpData
->FailedCpuList
!= NULL
) {
1907 (*CpuMpData
->FailedCpuList
)[ListIndex
] = END_OF_CPU_LIST
;
1911 return EFI_NOT_READY
;
1915 MP Initialize Library initialization.
1917 This service will allocate AP reset vector and wakeup all APs to do APs
1920 This service must be invoked before all other MP Initialize Library
1921 service are invoked.
1923 @retval EFI_SUCCESS MP initialization succeeds.
1924 @retval Others MP initialization fails.
1929 MpInitLibInitialize (
1933 CPU_MP_DATA
*OldCpuMpData
;
1934 CPU_INFO_IN_HOB
*CpuInfoInHob
;
1935 UINT32 MaxLogicalProcessorNumber
;
1937 MP_ASSEMBLY_ADDRESS_MAP AddressMap
;
1938 CPU_VOLATILE_REGISTERS VolatileRegisters
;
1940 UINT32 MonitorFilterSize
;
1943 CPU_MP_DATA
*CpuMpData
;
1945 UINT8
*MonitorBuffer
;
1947 UINTN ApResetVectorSize
;
1948 UINTN BackupBufferAddr
;
1951 OldCpuMpData
= GetCpuMpDataFromGuidedHob ();
1952 if (OldCpuMpData
== NULL
) {
1953 MaxLogicalProcessorNumber
= PcdGet32(PcdCpuMaxLogicalProcessorNumber
);
1955 MaxLogicalProcessorNumber
= OldCpuMpData
->CpuCount
;
1957 ASSERT (MaxLogicalProcessorNumber
!= 0);
1959 AsmGetAddressMap (&AddressMap
);
1960 ApResetVectorSize
= GetApResetVectorSize (&AddressMap
);
1961 ApStackSize
= PcdGet32(PcdCpuApStackSize
);
1962 ApLoopMode
= GetApLoopMode (&MonitorFilterSize
);
1965 // Save BSP's Control registers for APs.
1967 SaveVolatileRegisters (&VolatileRegisters
);
1969 BufferSize
= ApStackSize
* MaxLogicalProcessorNumber
;
1970 BufferSize
+= MonitorFilterSize
* MaxLogicalProcessorNumber
;
1971 BufferSize
+= ApResetVectorSize
;
1972 BufferSize
= ALIGN_VALUE (BufferSize
, 8);
1973 BufferSize
+= VolatileRegisters
.Idtr
.Limit
+ 1;
1974 BufferSize
+= sizeof (CPU_MP_DATA
);
1975 BufferSize
+= (sizeof (CPU_AP_DATA
) + sizeof (CPU_INFO_IN_HOB
))* MaxLogicalProcessorNumber
;
1976 MpBuffer
= AllocatePages (EFI_SIZE_TO_PAGES (BufferSize
));
1977 ASSERT (MpBuffer
!= NULL
);
1978 ZeroMem (MpBuffer
, BufferSize
);
1979 Buffer
= (UINTN
) MpBuffer
;
1982 // The layout of the Buffer is as below:
1984 // +--------------------+ <-- Buffer
1986 // +--------------------+ <-- MonitorBuffer
1987 // AP Monitor Filters (N)
1988 // +--------------------+ <-- BackupBufferAddr (CpuMpData->BackupBuffer)
1990 // +--------------------+
1992 // +--------------------+ <-- ApIdtBase (8-byte boundary)
1993 // AP IDT All APs share one separate IDT. So AP can get address of CPU_MP_DATA from IDT Base.
1994 // +--------------------+ <-- CpuMpData
1996 // +--------------------+ <-- CpuMpData->CpuData
1998 // +--------------------+ <-- CpuMpData->CpuInfoInHob
1999 // CPU_INFO_IN_HOB (N)
2000 // +--------------------+
2002 MonitorBuffer
= (UINT8
*) (Buffer
+ ApStackSize
* MaxLogicalProcessorNumber
);
2003 BackupBufferAddr
= (UINTN
) MonitorBuffer
+ MonitorFilterSize
* MaxLogicalProcessorNumber
;
2004 ApIdtBase
= ALIGN_VALUE (BackupBufferAddr
+ ApResetVectorSize
, 8);
2005 CpuMpData
= (CPU_MP_DATA
*) (ApIdtBase
+ VolatileRegisters
.Idtr
.Limit
+ 1);
2006 CpuMpData
->Buffer
= Buffer
;
2007 CpuMpData
->CpuApStackSize
= ApStackSize
;
2008 CpuMpData
->BackupBuffer
= BackupBufferAddr
;
2009 CpuMpData
->BackupBufferSize
= ApResetVectorSize
;
2010 CpuMpData
->WakeupBuffer
= (UINTN
) -1;
2011 CpuMpData
->CpuCount
= 1;
2012 CpuMpData
->BspNumber
= 0;
2013 CpuMpData
->WaitEvent
= NULL
;
2014 CpuMpData
->SwitchBspFlag
= FALSE
;
2015 CpuMpData
->CpuData
= (CPU_AP_DATA
*) (CpuMpData
+ 1);
2016 CpuMpData
->CpuInfoInHob
= (UINT64
) (UINTN
) (CpuMpData
->CpuData
+ MaxLogicalProcessorNumber
);
2017 InitializeSpinLock(&CpuMpData
->MpLock
);
2018 CpuMpData
->SevEsIsEnabled
= PcdGetBool (PcdSevEsIsEnabled
);
2019 CpuMpData
->SevEsAPBuffer
= (UINTN
) -1;
2020 CpuMpData
->GhcbBase
= PcdGet64 (PcdGhcbBase
);
2023 // Make sure no memory usage outside of the allocated buffer.
2025 ASSERT ((CpuMpData
->CpuInfoInHob
+ sizeof (CPU_INFO_IN_HOB
) * MaxLogicalProcessorNumber
) ==
2026 Buffer
+ BufferSize
);
2029 // Duplicate BSP's IDT to APs.
2030 // All APs share one separate IDT. So AP can get the address of CpuMpData by using IDTR.BASE + IDTR.LIMIT + 1
2032 CopyMem ((VOID
*)ApIdtBase
, (VOID
*)VolatileRegisters
.Idtr
.Base
, VolatileRegisters
.Idtr
.Limit
+ 1);
2033 VolatileRegisters
.Idtr
.Base
= ApIdtBase
;
2035 // Don't pass BSP's TR to APs to avoid AP init failure.
2037 VolatileRegisters
.Tr
= 0;
2038 CopyMem (&CpuMpData
->CpuData
[0].VolatileRegisters
, &VolatileRegisters
, sizeof (VolatileRegisters
));
2040 // Set BSP basic information
2042 InitializeApData (CpuMpData
, 0, 0, CpuMpData
->Buffer
+ ApStackSize
);
2044 // Save assembly code information
2046 CopyMem (&CpuMpData
->AddressMap
, &AddressMap
, sizeof (MP_ASSEMBLY_ADDRESS_MAP
));
2048 // Finally set AP loop mode
2050 CpuMpData
->ApLoopMode
= ApLoopMode
;
2051 DEBUG ((DEBUG_INFO
, "AP Loop Mode is %d\n", CpuMpData
->ApLoopMode
));
2053 CpuMpData
->WakeUpByInitSipiSipi
= (CpuMpData
->ApLoopMode
== ApInHltLoop
);
2056 // Set up APs wakeup signal buffer
2058 for (Index
= 0; Index
< MaxLogicalProcessorNumber
; Index
++) {
2059 CpuMpData
->CpuData
[Index
].StartupApSignal
=
2060 (UINT32
*)(MonitorBuffer
+ MonitorFilterSize
* Index
);
2063 // Enable the local APIC for Virtual Wire Mode.
2065 ProgramVirtualWireMode ();
2067 if (OldCpuMpData
== NULL
) {
2068 if (MaxLogicalProcessorNumber
> 1) {
2070 // Wakeup all APs and calculate the processor count in system
2072 CollectProcessorCount (CpuMpData
);
2076 // APs have been wakeup before, just get the CPU Information
2079 OldCpuMpData
->NewCpuMpData
= CpuMpData
;
2080 CpuMpData
->CpuCount
= OldCpuMpData
->CpuCount
;
2081 CpuMpData
->BspNumber
= OldCpuMpData
->BspNumber
;
2082 CpuMpData
->CpuInfoInHob
= OldCpuMpData
->CpuInfoInHob
;
2083 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
2084 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
2085 InitializeSpinLock(&CpuMpData
->CpuData
[Index
].ApLock
);
2086 CpuMpData
->CpuData
[Index
].CpuHealthy
= (CpuInfoInHob
[Index
].Health
== 0)? TRUE
:FALSE
;
2087 CpuMpData
->CpuData
[Index
].ApFunction
= 0;
2091 if (!GetMicrocodePatchInfoFromHob (
2092 &CpuMpData
->MicrocodePatchAddress
,
2093 &CpuMpData
->MicrocodePatchRegionSize
2096 // The microcode patch information cache HOB does not exist, which means
2097 // the microcode patches data has not been loaded into memory yet
2099 ShadowMicrocodeUpdatePatch (CpuMpData
);
2103 // Detect and apply Microcode on BSP
2105 MicrocodeDetect (CpuMpData
, CpuMpData
->BspNumber
);
2107 // Store BSP's MTRR setting
2109 MtrrGetAllMtrrs (&CpuMpData
->MtrrTable
);
2112 // Wakeup APs to do some AP initialize sync (Microcode & MTRR)
2114 if (CpuMpData
->CpuCount
> 1) {
2115 if (OldCpuMpData
!= NULL
) {
2117 // Only needs to use this flag for DXE phase to update the wake up
2118 // buffer. Wakeup buffer allocated in PEI phase is no longer valid
2121 CpuMpData
->InitFlag
= ApInitReconfig
;
2123 WakeUpAP (CpuMpData
, TRUE
, 0, ApInitializeSync
, CpuMpData
, TRUE
);
2125 // Wait for all APs finished initialization
2127 while (CpuMpData
->FinishedCount
< (CpuMpData
->CpuCount
- 1)) {
2130 if (OldCpuMpData
!= NULL
) {
2131 CpuMpData
->InitFlag
= ApInitDone
;
2133 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
2134 SetApState (&CpuMpData
->CpuData
[Index
], CpuStateIdle
);
2139 // Initialize global data for MP support
2141 InitMpGlobalData (CpuMpData
);
2147 Gets detailed MP-related information on the requested processor at the
2148 instant this call is made. This service may only be called from the BSP.
2150 @param[in] ProcessorNumber The handle number of processor.
2151 @param[out] ProcessorInfoBuffer A pointer to the buffer where information for
2152 the requested processor is deposited.
2153 @param[out] HealthData Return processor health data.
2155 @retval EFI_SUCCESS Processor information was returned.
2156 @retval EFI_DEVICE_ERROR The calling processor is an AP.
2157 @retval EFI_INVALID_PARAMETER ProcessorInfoBuffer is NULL.
2158 @retval EFI_NOT_FOUND The processor with the handle specified by
2159 ProcessorNumber does not exist in the platform.
2160 @retval EFI_NOT_READY MP Initialize Library is not initialized.
2165 MpInitLibGetProcessorInfo (
2166 IN UINTN ProcessorNumber
,
2167 OUT EFI_PROCESSOR_INFORMATION
*ProcessorInfoBuffer
,
2168 OUT EFI_HEALTH_FLAGS
*HealthData OPTIONAL
2171 CPU_MP_DATA
*CpuMpData
;
2173 CPU_INFO_IN_HOB
*CpuInfoInHob
;
2174 UINTN OriginalProcessorNumber
;
2176 CpuMpData
= GetCpuMpData ();
2177 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
2180 // Lower 24 bits contains the actual processor number.
2182 OriginalProcessorNumber
= ProcessorNumber
;
2183 ProcessorNumber
&= BIT24
- 1;
2186 // Check whether caller processor is BSP
2188 MpInitLibWhoAmI (&CallerNumber
);
2189 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2190 return EFI_DEVICE_ERROR
;
2193 if (ProcessorInfoBuffer
== NULL
) {
2194 return EFI_INVALID_PARAMETER
;
2197 if (ProcessorNumber
>= CpuMpData
->CpuCount
) {
2198 return EFI_NOT_FOUND
;
2201 ProcessorInfoBuffer
->ProcessorId
= (UINT64
) CpuInfoInHob
[ProcessorNumber
].ApicId
;
2202 ProcessorInfoBuffer
->StatusFlag
= 0;
2203 if (ProcessorNumber
== CpuMpData
->BspNumber
) {
2204 ProcessorInfoBuffer
->StatusFlag
|= PROCESSOR_AS_BSP_BIT
;
2206 if (CpuMpData
->CpuData
[ProcessorNumber
].CpuHealthy
) {
2207 ProcessorInfoBuffer
->StatusFlag
|= PROCESSOR_HEALTH_STATUS_BIT
;
2209 if (GetApState (&CpuMpData
->CpuData
[ProcessorNumber
]) == CpuStateDisabled
) {
2210 ProcessorInfoBuffer
->StatusFlag
&= ~PROCESSOR_ENABLED_BIT
;
2212 ProcessorInfoBuffer
->StatusFlag
|= PROCESSOR_ENABLED_BIT
;
2216 // Get processor location information
2218 GetProcessorLocationByApicId (
2219 CpuInfoInHob
[ProcessorNumber
].ApicId
,
2220 &ProcessorInfoBuffer
->Location
.Package
,
2221 &ProcessorInfoBuffer
->Location
.Core
,
2222 &ProcessorInfoBuffer
->Location
.Thread
2225 if ((OriginalProcessorNumber
& CPU_V2_EXTENDED_TOPOLOGY
) != 0) {
2226 GetProcessorLocation2ByApicId (
2227 CpuInfoInHob
[ProcessorNumber
].ApicId
,
2228 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Package
,
2229 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Die
,
2230 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Tile
,
2231 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Module
,
2232 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Core
,
2233 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Thread
2237 if (HealthData
!= NULL
) {
2238 HealthData
->Uint32
= CpuInfoInHob
[ProcessorNumber
].Health
;
2245 Worker function to switch the requested AP to be the BSP from that point onward.
2247 @param[in] ProcessorNumber The handle number of AP that is to become the new BSP.
2248 @param[in] EnableOldBSP If TRUE, then the old BSP will be listed as an
2249 enabled AP. Otherwise, it will be disabled.
2251 @retval EFI_SUCCESS BSP successfully switched.
2252 @retval others Failed to switch BSP.
2257 IN UINTN ProcessorNumber
,
2258 IN BOOLEAN EnableOldBSP
2261 CPU_MP_DATA
*CpuMpData
;
2264 MSR_IA32_APIC_BASE_REGISTER ApicBaseMsr
;
2265 BOOLEAN OldInterruptState
;
2266 BOOLEAN OldTimerInterruptState
;
2269 // Save and Disable Local APIC timer interrupt
2271 OldTimerInterruptState
= GetApicTimerInterruptState ();
2272 DisableApicTimerInterrupt ();
2274 // Before send both BSP and AP to a procedure to exchange their roles,
2275 // interrupt must be disabled. This is because during the exchange role
2276 // process, 2 CPU may use 1 stack. If interrupt happens, the stack will
2277 // be corrupted, since interrupt return address will be pushed to stack
2280 OldInterruptState
= SaveAndDisableInterrupts ();
2283 // Mask LINT0 & LINT1 for the old BSP
2285 DisableLvtInterrupts ();
2287 CpuMpData
= GetCpuMpData ();
2290 // Check whether caller processor is BSP
2292 MpInitLibWhoAmI (&CallerNumber
);
2293 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2294 return EFI_DEVICE_ERROR
;
2297 if (ProcessorNumber
>= CpuMpData
->CpuCount
) {
2298 return EFI_NOT_FOUND
;
2302 // Check whether specified AP is disabled
2304 State
= GetApState (&CpuMpData
->CpuData
[ProcessorNumber
]);
2305 if (State
== CpuStateDisabled
) {
2306 return EFI_INVALID_PARAMETER
;
2310 // Check whether ProcessorNumber specifies the current BSP
2312 if (ProcessorNumber
== CpuMpData
->BspNumber
) {
2313 return EFI_INVALID_PARAMETER
;
2317 // Check whether specified AP is busy
2319 if (State
== CpuStateBusy
) {
2320 return EFI_NOT_READY
;
2323 CpuMpData
->BSPInfo
.State
= CPU_SWITCH_STATE_IDLE
;
2324 CpuMpData
->APInfo
.State
= CPU_SWITCH_STATE_IDLE
;
2325 CpuMpData
->SwitchBspFlag
= TRUE
;
2326 CpuMpData
->NewBspNumber
= ProcessorNumber
;
2329 // Clear the BSP bit of MSR_IA32_APIC_BASE
2331 ApicBaseMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_APIC_BASE
);
2332 ApicBaseMsr
.Bits
.BSP
= 0;
2333 AsmWriteMsr64 (MSR_IA32_APIC_BASE
, ApicBaseMsr
.Uint64
);
2336 // Need to wakeUp AP (future BSP).
2338 WakeUpAP (CpuMpData
, FALSE
, ProcessorNumber
, FutureBSPProc
, CpuMpData
, TRUE
);
2340 AsmExchangeRole (&CpuMpData
->BSPInfo
, &CpuMpData
->APInfo
);
2343 // Set the BSP bit of MSR_IA32_APIC_BASE on new BSP
2345 ApicBaseMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_APIC_BASE
);
2346 ApicBaseMsr
.Bits
.BSP
= 1;
2347 AsmWriteMsr64 (MSR_IA32_APIC_BASE
, ApicBaseMsr
.Uint64
);
2348 ProgramVirtualWireMode ();
2351 // Wait for old BSP finished AP task
2353 while (GetApState (&CpuMpData
->CpuData
[CallerNumber
]) != CpuStateFinished
) {
2357 CpuMpData
->SwitchBspFlag
= FALSE
;
2359 // Set old BSP enable state
2361 if (!EnableOldBSP
) {
2362 SetApState (&CpuMpData
->CpuData
[CallerNumber
], CpuStateDisabled
);
2364 SetApState (&CpuMpData
->CpuData
[CallerNumber
], CpuStateIdle
);
2367 // Save new BSP number
2369 CpuMpData
->BspNumber
= (UINT32
) ProcessorNumber
;
2372 // Restore interrupt state.
2374 SetInterruptState (OldInterruptState
);
2376 if (OldTimerInterruptState
) {
2377 EnableApicTimerInterrupt ();
2384 Worker function to let the caller enable or disable an AP from this point onward.
2385 This service may only be called from the BSP.
2387 @param[in] ProcessorNumber The handle number of AP.
2388 @param[in] EnableAP Specifies the new state for the processor for
2389 enabled, FALSE for disabled.
2390 @param[in] HealthFlag If not NULL, a pointer to a value that specifies
2391 the new health status of the AP.
2393 @retval EFI_SUCCESS The specified AP was enabled or disabled successfully.
2394 @retval others Failed to Enable/Disable AP.
2398 EnableDisableApWorker (
2399 IN UINTN ProcessorNumber
,
2400 IN BOOLEAN EnableAP
,
2401 IN UINT32
*HealthFlag OPTIONAL
2404 CPU_MP_DATA
*CpuMpData
;
2407 CpuMpData
= GetCpuMpData ();
2410 // Check whether caller processor is BSP
2412 MpInitLibWhoAmI (&CallerNumber
);
2413 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2414 return EFI_DEVICE_ERROR
;
2417 if (ProcessorNumber
== CpuMpData
->BspNumber
) {
2418 return EFI_INVALID_PARAMETER
;
2421 if (ProcessorNumber
>= CpuMpData
->CpuCount
) {
2422 return EFI_NOT_FOUND
;
2426 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateDisabled
);
2428 ResetProcessorToIdleState (ProcessorNumber
);
2431 if (HealthFlag
!= NULL
) {
2432 CpuMpData
->CpuData
[ProcessorNumber
].CpuHealthy
=
2433 (BOOLEAN
) ((*HealthFlag
& PROCESSOR_HEALTH_STATUS_BIT
) != 0);
2440 This return the handle number for the calling processor. This service may be
2441 called from the BSP and APs.
2443 @param[out] ProcessorNumber Pointer to the handle number of AP.
2444 The range is from 0 to the total number of
2445 logical processors minus 1. The total number of
2446 logical processors can be retrieved by
2447 MpInitLibGetNumberOfProcessors().
2449 @retval EFI_SUCCESS The current processor handle number was returned
2451 @retval EFI_INVALID_PARAMETER ProcessorNumber is NULL.
2452 @retval EFI_NOT_READY MP Initialize Library is not initialized.
2458 OUT UINTN
*ProcessorNumber
2461 CPU_MP_DATA
*CpuMpData
;
2463 if (ProcessorNumber
== NULL
) {
2464 return EFI_INVALID_PARAMETER
;
2467 CpuMpData
= GetCpuMpData ();
2469 return GetProcessorNumber (CpuMpData
, ProcessorNumber
);
2473 Retrieves the number of logical processor in the platform and the number of
2474 those logical processors that are enabled on this boot. This service may only
2475 be called from the BSP.
2477 @param[out] NumberOfProcessors Pointer to the total number of logical
2478 processors in the system, including the BSP
2480 @param[out] NumberOfEnabledProcessors Pointer to the number of enabled logical
2481 processors that exist in system, including
2484 @retval EFI_SUCCESS The number of logical processors and enabled
2485 logical processors was retrieved.
2486 @retval EFI_DEVICE_ERROR The calling processor is an AP.
2487 @retval EFI_INVALID_PARAMETER NumberOfProcessors is NULL and NumberOfEnabledProcessors
2489 @retval EFI_NOT_READY MP Initialize Library is not initialized.
2494 MpInitLibGetNumberOfProcessors (
2495 OUT UINTN
*NumberOfProcessors
, OPTIONAL
2496 OUT UINTN
*NumberOfEnabledProcessors OPTIONAL
2499 CPU_MP_DATA
*CpuMpData
;
2501 UINTN ProcessorNumber
;
2502 UINTN EnabledProcessorNumber
;
2505 CpuMpData
= GetCpuMpData ();
2507 if ((NumberOfProcessors
== NULL
) && (NumberOfEnabledProcessors
== NULL
)) {
2508 return EFI_INVALID_PARAMETER
;
2512 // Check whether caller processor is BSP
2514 MpInitLibWhoAmI (&CallerNumber
);
2515 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2516 return EFI_DEVICE_ERROR
;
2519 ProcessorNumber
= CpuMpData
->CpuCount
;
2520 EnabledProcessorNumber
= 0;
2521 for (Index
= 0; Index
< ProcessorNumber
; Index
++) {
2522 if (GetApState (&CpuMpData
->CpuData
[Index
]) != CpuStateDisabled
) {
2523 EnabledProcessorNumber
++;
2527 if (NumberOfProcessors
!= NULL
) {
2528 *NumberOfProcessors
= ProcessorNumber
;
2530 if (NumberOfEnabledProcessors
!= NULL
) {
2531 *NumberOfEnabledProcessors
= EnabledProcessorNumber
;
2539 Worker function to execute a caller provided function on all enabled APs.
2541 @param[in] Procedure A pointer to the function to be run on
2542 enabled APs of the system.
2543 @param[in] SingleThread If TRUE, then all the enabled APs execute
2544 the function specified by Procedure one by
2545 one, in ascending order of processor handle
2546 number. If FALSE, then all the enabled APs
2547 execute the function specified by Procedure
2549 @param[in] ExcludeBsp Whether let BSP also trig this task.
2550 @param[in] WaitEvent The event created by the caller with CreateEvent()
2552 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
2553 APs to return from Procedure, either for
2554 blocking or non-blocking mode.
2555 @param[in] ProcedureArgument The parameter passed into Procedure for
2557 @param[out] FailedCpuList If all APs finish successfully, then its
2558 content is set to NULL. If not all APs
2559 finish before timeout expires, then its
2560 content is set to address of the buffer
2561 holding handle numbers of the failed APs.
2563 @retval EFI_SUCCESS In blocking mode, all APs have finished before
2564 the timeout expired.
2565 @retval EFI_SUCCESS In non-blocking mode, function has been dispatched
2567 @retval others Failed to Startup all APs.
2571 StartupAllCPUsWorker (
2572 IN EFI_AP_PROCEDURE Procedure
,
2573 IN BOOLEAN SingleThread
,
2574 IN BOOLEAN ExcludeBsp
,
2575 IN EFI_EVENT WaitEvent OPTIONAL
,
2576 IN UINTN TimeoutInMicroseconds
,
2577 IN VOID
*ProcedureArgument OPTIONAL
,
2578 OUT UINTN
**FailedCpuList OPTIONAL
2582 CPU_MP_DATA
*CpuMpData
;
2583 UINTN ProcessorCount
;
2584 UINTN ProcessorNumber
;
2586 CPU_AP_DATA
*CpuData
;
2587 BOOLEAN HasEnabledAp
;
2590 CpuMpData
= GetCpuMpData ();
2592 if (FailedCpuList
!= NULL
) {
2593 *FailedCpuList
= NULL
;
2596 if (CpuMpData
->CpuCount
== 1 && ExcludeBsp
) {
2597 return EFI_NOT_STARTED
;
2600 if (Procedure
== NULL
) {
2601 return EFI_INVALID_PARAMETER
;
2605 // Check whether caller processor is BSP
2607 MpInitLibWhoAmI (&CallerNumber
);
2608 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2609 return EFI_DEVICE_ERROR
;
2615 CheckAndUpdateApsStatus ();
2617 ProcessorCount
= CpuMpData
->CpuCount
;
2618 HasEnabledAp
= FALSE
;
2620 // Check whether all enabled APs are idle.
2621 // If any enabled AP is not idle, return EFI_NOT_READY.
2623 for (ProcessorNumber
= 0; ProcessorNumber
< ProcessorCount
; ProcessorNumber
++) {
2624 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
2625 if (ProcessorNumber
!= CpuMpData
->BspNumber
) {
2626 ApState
= GetApState (CpuData
);
2627 if (ApState
!= CpuStateDisabled
) {
2628 HasEnabledAp
= TRUE
;
2629 if (ApState
!= CpuStateIdle
) {
2631 // If any enabled APs are busy, return EFI_NOT_READY.
2633 return EFI_NOT_READY
;
2639 if (!HasEnabledAp
&& ExcludeBsp
) {
2641 // If no enabled AP exists and not include Bsp to do the procedure, return EFI_NOT_STARTED.
2643 return EFI_NOT_STARTED
;
2646 CpuMpData
->RunningCount
= 0;
2647 for (ProcessorNumber
= 0; ProcessorNumber
< ProcessorCount
; ProcessorNumber
++) {
2648 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
2649 CpuData
->Waiting
= FALSE
;
2650 if (ProcessorNumber
!= CpuMpData
->BspNumber
) {
2651 if (CpuData
->State
== CpuStateIdle
) {
2653 // Mark this processor as responsible for current calling.
2655 CpuData
->Waiting
= TRUE
;
2656 CpuMpData
->RunningCount
++;
2661 CpuMpData
->Procedure
= Procedure
;
2662 CpuMpData
->ProcArguments
= ProcedureArgument
;
2663 CpuMpData
->SingleThread
= SingleThread
;
2664 CpuMpData
->FinishedCount
= 0;
2665 CpuMpData
->FailedCpuList
= FailedCpuList
;
2666 CpuMpData
->ExpectedTime
= CalculateTimeout (
2667 TimeoutInMicroseconds
,
2668 &CpuMpData
->CurrentTime
2670 CpuMpData
->TotalTime
= 0;
2671 CpuMpData
->WaitEvent
= WaitEvent
;
2673 if (!SingleThread
) {
2674 WakeUpAP (CpuMpData
, TRUE
, 0, Procedure
, ProcedureArgument
, FALSE
);
2676 for (ProcessorNumber
= 0; ProcessorNumber
< ProcessorCount
; ProcessorNumber
++) {
2677 if (ProcessorNumber
== CallerNumber
) {
2680 if (CpuMpData
->CpuData
[ProcessorNumber
].Waiting
) {
2681 WakeUpAP (CpuMpData
, FALSE
, ProcessorNumber
, Procedure
, ProcedureArgument
, TRUE
);
2691 Procedure (ProcedureArgument
);
2694 Status
= EFI_SUCCESS
;
2695 if (WaitEvent
== NULL
) {
2697 Status
= CheckAllAPs ();
2698 } while (Status
== EFI_NOT_READY
);
2705 Worker function to let the caller get one enabled AP to execute a caller-provided
2708 @param[in] Procedure A pointer to the function to be run on
2709 enabled APs of the system.
2710 @param[in] ProcessorNumber The handle number of the AP.
2711 @param[in] WaitEvent The event created by the caller with CreateEvent()
2713 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
2714 APs to return from Procedure, either for
2715 blocking or non-blocking mode.
2716 @param[in] ProcedureArgument The parameter passed into Procedure for
2718 @param[out] Finished If AP returns from Procedure before the
2719 timeout expires, its content is set to TRUE.
2720 Otherwise, the value is set to FALSE.
2722 @retval EFI_SUCCESS In blocking mode, specified AP finished before
2723 the timeout expires.
2724 @retval others Failed to Startup AP.
2728 StartupThisAPWorker (
2729 IN EFI_AP_PROCEDURE Procedure
,
2730 IN UINTN ProcessorNumber
,
2731 IN EFI_EVENT WaitEvent OPTIONAL
,
2732 IN UINTN TimeoutInMicroseconds
,
2733 IN VOID
*ProcedureArgument OPTIONAL
,
2734 OUT BOOLEAN
*Finished OPTIONAL
2738 CPU_MP_DATA
*CpuMpData
;
2739 CPU_AP_DATA
*CpuData
;
2742 CpuMpData
= GetCpuMpData ();
2744 if (Finished
!= NULL
) {
2749 // Check whether caller processor is BSP
2751 MpInitLibWhoAmI (&CallerNumber
);
2752 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2753 return EFI_DEVICE_ERROR
;
2757 // Check whether processor with the handle specified by ProcessorNumber exists
2759 if (ProcessorNumber
>= CpuMpData
->CpuCount
) {
2760 return EFI_NOT_FOUND
;
2764 // Check whether specified processor is BSP
2766 if (ProcessorNumber
== CpuMpData
->BspNumber
) {
2767 return EFI_INVALID_PARAMETER
;
2771 // Check parameter Procedure
2773 if (Procedure
== NULL
) {
2774 return EFI_INVALID_PARAMETER
;
2780 CheckAndUpdateApsStatus ();
2783 // Check whether specified AP is disabled
2785 if (GetApState (&CpuMpData
->CpuData
[ProcessorNumber
]) == CpuStateDisabled
) {
2786 return EFI_INVALID_PARAMETER
;
2790 // If WaitEvent is not NULL, execute in non-blocking mode.
2791 // BSP saves data for CheckAPsStatus(), and returns EFI_SUCCESS.
2792 // CheckAPsStatus() will check completion and timeout periodically.
2794 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
2795 CpuData
->WaitEvent
= WaitEvent
;
2796 CpuData
->Finished
= Finished
;
2797 CpuData
->ExpectedTime
= CalculateTimeout (TimeoutInMicroseconds
, &CpuData
->CurrentTime
);
2798 CpuData
->TotalTime
= 0;
2800 WakeUpAP (CpuMpData
, FALSE
, ProcessorNumber
, Procedure
, ProcedureArgument
, TRUE
);
2803 // If WaitEvent is NULL, execute in blocking mode.
2804 // BSP checks AP's state until it finishes or TimeoutInMicrosecsond expires.
2806 Status
= EFI_SUCCESS
;
2807 if (WaitEvent
== NULL
) {
2809 Status
= CheckThisAP (ProcessorNumber
);
2810 } while (Status
== EFI_NOT_READY
);
2817 Get pointer to CPU MP Data structure from GUIDed HOB.
2819 @return The pointer to CPU MP Data structure.
2822 GetCpuMpDataFromGuidedHob (
2826 EFI_HOB_GUID_TYPE
*GuidHob
;
2828 CPU_MP_DATA
*CpuMpData
;
2831 GuidHob
= GetFirstGuidHob (&mCpuInitMpLibHobGuid
);
2832 if (GuidHob
!= NULL
) {
2833 DataInHob
= GET_GUID_HOB_DATA (GuidHob
);
2834 CpuMpData
= (CPU_MP_DATA
*) (*(UINTN
*) DataInHob
);
2840 This service executes a caller provided function on all enabled CPUs.
2842 @param[in] Procedure A pointer to the function to be run on
2843 enabled APs of the system. See type
2845 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
2846 APs to return from Procedure, either for
2847 blocking or non-blocking mode. Zero means
2848 infinity. TimeoutInMicroseconds is ignored
2850 @param[in] ProcedureArgument The parameter passed into Procedure for
2853 @retval EFI_SUCCESS In blocking mode, all CPUs have finished before
2854 the timeout expired.
2855 @retval EFI_SUCCESS In non-blocking mode, function has been dispatched
2856 to all enabled CPUs.
2857 @retval EFI_DEVICE_ERROR Caller processor is AP.
2858 @retval EFI_NOT_READY Any enabled APs are busy.
2859 @retval EFI_NOT_READY MP Initialize Library is not initialized.
2860 @retval EFI_TIMEOUT In blocking mode, the timeout expired before
2861 all enabled APs have finished.
2862 @retval EFI_INVALID_PARAMETER Procedure is NULL.
2867 MpInitLibStartupAllCPUs (
2868 IN EFI_AP_PROCEDURE Procedure
,
2869 IN UINTN TimeoutInMicroseconds
,
2870 IN VOID
*ProcedureArgument OPTIONAL
2873 return StartupAllCPUsWorker (
2878 TimeoutInMicroseconds
,