2 CPU MP Initialize Library common functions.
4 Copyright (c) 2016 - 2020, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2020, AMD Inc. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
12 #include <Library/VmgExitLib.h>
13 #include <Register/Amd/Fam17Msr.h>
14 #include <Register/Amd/Ghcb.h>
16 EFI_GUID mCpuInitMpLibHobGuid
= CPU_INIT_MP_LIB_HOB_GUID
;
20 The function will check if BSP Execute Disable is enabled.
22 DxeIpl may have enabled Execute Disable for BSP, APs need to
23 get the status and sync up the settings.
24 If BSP's CR0.Paging is not set, BSP execute Disble feature is
27 @retval TRUE BSP Execute Disable is enabled.
28 @retval FALSE BSP Execute Disable is not enabled.
31 IsBspExecuteDisableEnabled (
36 CPUID_EXTENDED_CPU_SIG_EDX Edx
;
37 MSR_IA32_EFER_REGISTER EferMsr
;
42 Cr0
.UintN
= AsmReadCr0 ();
43 if (Cr0
.Bits
.PG
!= 0) {
45 // If CR0 Paging bit is set
47 AsmCpuid (CPUID_EXTENDED_FUNCTION
, &Eax
, NULL
, NULL
, NULL
);
48 if (Eax
>= CPUID_EXTENDED_CPU_SIG
) {
49 AsmCpuid (CPUID_EXTENDED_CPU_SIG
, NULL
, NULL
, NULL
, &Edx
.Uint32
);
52 // Bit 20: Execute Disable Bit available.
54 if (Edx
.Bits
.NX
!= 0) {
55 EferMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_EFER
);
58 // Bit 11: Execute Disable Bit enable.
60 if (EferMsr
.Bits
.NXE
!= 0) {
71 Worker function for SwitchBSP().
73 Worker function for SwitchBSP(), assigned to the AP which is intended
76 @param[in] Buffer Pointer to CPU MP Data
84 CPU_MP_DATA
*DataInHob
;
86 DataInHob
= (CPU_MP_DATA
*) Buffer
;
87 AsmExchangeRole (&DataInHob
->APInfo
, &DataInHob
->BSPInfo
);
91 Get the Application Processors state.
93 @param[in] CpuData The pointer to CPU_AP_DATA of specified AP
99 IN CPU_AP_DATA
*CpuData
102 return CpuData
->State
;
106 Set the Application Processors state.
108 @param[in] CpuData The pointer to CPU_AP_DATA of specified AP
109 @param[in] State The AP status
113 IN CPU_AP_DATA
*CpuData
,
117 AcquireSpinLock (&CpuData
->ApLock
);
118 CpuData
->State
= State
;
119 ReleaseSpinLock (&CpuData
->ApLock
);
123 Save BSP's local APIC timer setting.
125 @param[in] CpuMpData Pointer to CPU MP Data
128 SaveLocalApicTimerSetting (
129 IN CPU_MP_DATA
*CpuMpData
133 // Record the current local APIC timer setting of BSP
136 &CpuMpData
->DivideValue
,
137 &CpuMpData
->PeriodicMode
,
140 CpuMpData
->CurrentTimerCount
= GetApicTimerCurrentCount ();
141 CpuMpData
->TimerInterruptState
= GetApicTimerInterruptState ();
145 Sync local APIC timer setting from BSP to AP.
147 @param[in] CpuMpData Pointer to CPU MP Data
150 SyncLocalApicTimerSetting (
151 IN CPU_MP_DATA
*CpuMpData
155 // Sync local APIC timer setting from BSP to AP
157 InitializeApicTimer (
158 CpuMpData
->DivideValue
,
159 CpuMpData
->CurrentTimerCount
,
160 CpuMpData
->PeriodicMode
,
164 // Disable AP's local APIC timer interrupt
166 DisableApicTimerInterrupt ();
170 Save the volatile registers required to be restored following INIT IPI.
172 @param[out] VolatileRegisters Returns buffer saved the volatile resisters
175 SaveVolatileRegisters (
176 OUT CPU_VOLATILE_REGISTERS
*VolatileRegisters
179 CPUID_VERSION_INFO_EDX VersionInfoEdx
;
181 VolatileRegisters
->Cr0
= AsmReadCr0 ();
182 VolatileRegisters
->Cr3
= AsmReadCr3 ();
183 VolatileRegisters
->Cr4
= AsmReadCr4 ();
185 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, NULL
, &VersionInfoEdx
.Uint32
);
186 if (VersionInfoEdx
.Bits
.DE
!= 0) {
188 // If processor supports Debugging Extensions feature
189 // by CPUID.[EAX=01H]:EDX.BIT2
191 VolatileRegisters
->Dr0
= AsmReadDr0 ();
192 VolatileRegisters
->Dr1
= AsmReadDr1 ();
193 VolatileRegisters
->Dr2
= AsmReadDr2 ();
194 VolatileRegisters
->Dr3
= AsmReadDr3 ();
195 VolatileRegisters
->Dr6
= AsmReadDr6 ();
196 VolatileRegisters
->Dr7
= AsmReadDr7 ();
199 AsmReadGdtr (&VolatileRegisters
->Gdtr
);
200 AsmReadIdtr (&VolatileRegisters
->Idtr
);
201 VolatileRegisters
->Tr
= AsmReadTr ();
205 Restore the volatile registers following INIT IPI.
207 @param[in] VolatileRegisters Pointer to volatile resisters
208 @param[in] IsRestoreDr TRUE: Restore DRx if supported
209 FALSE: Do not restore DRx
212 RestoreVolatileRegisters (
213 IN CPU_VOLATILE_REGISTERS
*VolatileRegisters
,
214 IN BOOLEAN IsRestoreDr
217 CPUID_VERSION_INFO_EDX VersionInfoEdx
;
218 IA32_TSS_DESCRIPTOR
*Tss
;
220 AsmWriteCr3 (VolatileRegisters
->Cr3
);
221 AsmWriteCr4 (VolatileRegisters
->Cr4
);
222 AsmWriteCr0 (VolatileRegisters
->Cr0
);
225 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, NULL
, &VersionInfoEdx
.Uint32
);
226 if (VersionInfoEdx
.Bits
.DE
!= 0) {
228 // If processor supports Debugging Extensions feature
229 // by CPUID.[EAX=01H]:EDX.BIT2
231 AsmWriteDr0 (VolatileRegisters
->Dr0
);
232 AsmWriteDr1 (VolatileRegisters
->Dr1
);
233 AsmWriteDr2 (VolatileRegisters
->Dr2
);
234 AsmWriteDr3 (VolatileRegisters
->Dr3
);
235 AsmWriteDr6 (VolatileRegisters
->Dr6
);
236 AsmWriteDr7 (VolatileRegisters
->Dr7
);
240 AsmWriteGdtr (&VolatileRegisters
->Gdtr
);
241 AsmWriteIdtr (&VolatileRegisters
->Idtr
);
242 if (VolatileRegisters
->Tr
!= 0 &&
243 VolatileRegisters
->Tr
< VolatileRegisters
->Gdtr
.Limit
) {
244 Tss
= (IA32_TSS_DESCRIPTOR
*)(VolatileRegisters
->Gdtr
.Base
+
245 VolatileRegisters
->Tr
);
246 if (Tss
->Bits
.P
== 1) {
247 Tss
->Bits
.Type
&= 0xD; // 1101 - Clear busy bit just in case
248 AsmWriteTr (VolatileRegisters
->Tr
);
254 Detect whether Mwait-monitor feature is supported.
256 @retval TRUE Mwait-monitor feature is supported.
257 @retval FALSE Mwait-monitor feature is not supported.
264 CPUID_VERSION_INFO_ECX VersionInfoEcx
;
266 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, &VersionInfoEcx
.Uint32
, NULL
);
267 return (VersionInfoEcx
.Bits
.MONITOR
== 1) ? TRUE
: FALSE
;
273 @param[out] MonitorFilterSize Returns the largest monitor-line size in bytes.
275 @return The AP loop mode.
279 OUT UINT32
*MonitorFilterSize
283 CPUID_MONITOR_MWAIT_EBX MonitorMwaitEbx
;
285 ASSERT (MonitorFilterSize
!= NULL
);
287 ApLoopMode
= PcdGet8 (PcdCpuApLoopMode
);
288 ASSERT (ApLoopMode
>= ApInHltLoop
&& ApLoopMode
<= ApInRunLoop
);
289 if (ApLoopMode
== ApInMwaitLoop
) {
290 if (!IsMwaitSupport ()) {
292 // If processor does not support MONITOR/MWAIT feature,
293 // force AP in Hlt-loop mode
295 ApLoopMode
= ApInHltLoop
;
298 if (PcdGetBool (PcdSevEsIsEnabled
)) {
300 // For SEV-ES, force AP in Hlt-loop mode in order to use the GHCB
301 // protocol for starting APs
303 ApLoopMode
= ApInHltLoop
;
307 if (ApLoopMode
!= ApInMwaitLoop
) {
308 *MonitorFilterSize
= sizeof (UINT32
);
311 // CPUID.[EAX=05H]:EBX.BIT0-15: Largest monitor-line size in bytes
312 // CPUID.[EAX=05H].EDX: C-states supported using MWAIT
314 AsmCpuid (CPUID_MONITOR_MWAIT
, NULL
, &MonitorMwaitEbx
.Uint32
, NULL
, NULL
);
315 *MonitorFilterSize
= MonitorMwaitEbx
.Bits
.LargestMonitorLineSize
;
322 Sort the APIC ID of all processors.
324 This function sorts the APIC ID of all processors so that processor number is
325 assigned in the ascending order of APIC ID which eases MP debugging.
327 @param[in] CpuMpData Pointer to PEI CPU MP Data
331 IN CPU_MP_DATA
*CpuMpData
338 CPU_INFO_IN_HOB CpuInfo
;
340 CPU_INFO_IN_HOB
*CpuInfoInHob
;
341 volatile UINT32
*StartupApSignal
;
343 ApCount
= CpuMpData
->CpuCount
- 1;
344 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
346 for (Index1
= 0; Index1
< ApCount
; Index1
++) {
349 // Sort key is the hardware default APIC ID
351 ApicId
= CpuInfoInHob
[Index1
].ApicId
;
352 for (Index2
= Index1
+ 1; Index2
<= ApCount
; Index2
++) {
353 if (ApicId
> CpuInfoInHob
[Index2
].ApicId
) {
355 ApicId
= CpuInfoInHob
[Index2
].ApicId
;
358 if (Index3
!= Index1
) {
359 CopyMem (&CpuInfo
, &CpuInfoInHob
[Index3
], sizeof (CPU_INFO_IN_HOB
));
361 &CpuInfoInHob
[Index3
],
362 &CpuInfoInHob
[Index1
],
363 sizeof (CPU_INFO_IN_HOB
)
365 CopyMem (&CpuInfoInHob
[Index1
], &CpuInfo
, sizeof (CPU_INFO_IN_HOB
));
368 // Also exchange the StartupApSignal.
370 StartupApSignal
= CpuMpData
->CpuData
[Index3
].StartupApSignal
;
371 CpuMpData
->CpuData
[Index3
].StartupApSignal
=
372 CpuMpData
->CpuData
[Index1
].StartupApSignal
;
373 CpuMpData
->CpuData
[Index1
].StartupApSignal
= StartupApSignal
;
378 // Get the processor number for the BSP
380 ApicId
= GetInitialApicId ();
381 for (Index1
= 0; Index1
< CpuMpData
->CpuCount
; Index1
++) {
382 if (CpuInfoInHob
[Index1
].ApicId
== ApicId
) {
383 CpuMpData
->BspNumber
= (UINT32
) Index1
;
391 Enable x2APIC mode on APs.
393 @param[in, out] Buffer Pointer to private data buffer.
401 SetApicMode (LOCAL_APIC_MODE_X2APIC
);
407 @param[in, out] Buffer Pointer to private data buffer.
415 CPU_MP_DATA
*CpuMpData
;
416 UINTN ProcessorNumber
;
419 CpuMpData
= (CPU_MP_DATA
*) Buffer
;
420 Status
= GetProcessorNumber (CpuMpData
, &ProcessorNumber
);
421 ASSERT_EFI_ERROR (Status
);
423 // Load microcode on AP
425 MicrocodeDetect (CpuMpData
, ProcessorNumber
);
427 // Sync BSP's MTRR table to AP
429 MtrrSetAllMtrrs (&CpuMpData
->MtrrTable
);
433 Find the current Processor number by APIC ID.
435 @param[in] CpuMpData Pointer to PEI CPU MP Data
436 @param[out] ProcessorNumber Return the pocessor number found
438 @retval EFI_SUCCESS ProcessorNumber is found and returned.
439 @retval EFI_NOT_FOUND ProcessorNumber is not found.
443 IN CPU_MP_DATA
*CpuMpData
,
444 OUT UINTN
*ProcessorNumber
447 UINTN TotalProcessorNumber
;
449 CPU_INFO_IN_HOB
*CpuInfoInHob
;
450 UINT32 CurrentApicId
;
452 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
454 TotalProcessorNumber
= CpuMpData
->CpuCount
;
455 CurrentApicId
= GetApicId ();
456 for (Index
= 0; Index
< TotalProcessorNumber
; Index
++) {
457 if (CpuInfoInHob
[Index
].ApicId
== CurrentApicId
) {
458 *ProcessorNumber
= Index
;
463 return EFI_NOT_FOUND
;
467 This function will get CPU count in the system.
469 @param[in] CpuMpData Pointer to PEI CPU MP Data
471 @return CPU count detected
474 CollectProcessorCount (
475 IN CPU_MP_DATA
*CpuMpData
479 CPU_INFO_IN_HOB
*CpuInfoInHob
;
483 // Send 1st broadcast IPI to APs to wakeup APs
485 CpuMpData
->InitFlag
= ApInitConfig
;
486 WakeUpAP (CpuMpData
, TRUE
, 0, NULL
, NULL
, TRUE
);
487 CpuMpData
->InitFlag
= ApInitDone
;
488 ASSERT (CpuMpData
->CpuCount
<= PcdGet32 (PcdCpuMaxLogicalProcessorNumber
));
490 // Wait for all APs finished the initialization
492 while (CpuMpData
->FinishedCount
< (CpuMpData
->CpuCount
- 1)) {
498 // Enable x2APIC mode if
499 // 1. Number of CPU is greater than 255; or
500 // 2. There are any logical processors reporting an Initial APIC ID of 255 or greater.
503 if (CpuMpData
->CpuCount
> 255) {
505 // If there are more than 255 processor found, force to enable X2APIC
509 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
510 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
511 if (CpuInfoInHob
[Index
].InitialApicId
>= 0xFF) {
519 DEBUG ((DEBUG_INFO
, "Force x2APIC mode!\n"));
521 // Wakeup all APs to enable x2APIC mode
523 WakeUpAP (CpuMpData
, TRUE
, 0, ApFuncEnableX2Apic
, NULL
, TRUE
);
525 // Wait for all known APs finished
527 while (CpuMpData
->FinishedCount
< (CpuMpData
->CpuCount
- 1)) {
531 // Enable x2APIC on BSP
533 SetApicMode (LOCAL_APIC_MODE_X2APIC
);
535 // Set BSP/Aps state to IDLE
537 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
538 SetApState (&CpuMpData
->CpuData
[Index
], CpuStateIdle
);
541 DEBUG ((DEBUG_INFO
, "APIC MODE is %d\n", GetApicMode ()));
543 // Sort BSP/Aps by CPU APIC ID in ascending order
545 SortApicId (CpuMpData
);
547 DEBUG ((DEBUG_INFO
, "MpInitLib: Find %d processors in system.\n", CpuMpData
->CpuCount
));
549 return CpuMpData
->CpuCount
;
553 Initialize CPU AP Data when AP is wakeup at the first time.
555 @param[in, out] CpuMpData Pointer to PEI CPU MP Data
556 @param[in] ProcessorNumber The handle number of processor
557 @param[in] BistData Processor BIST data
558 @param[in] ApTopOfStack Top of AP stack
563 IN OUT CPU_MP_DATA
*CpuMpData
,
564 IN UINTN ProcessorNumber
,
566 IN UINT64 ApTopOfStack
569 CPU_INFO_IN_HOB
*CpuInfoInHob
;
570 MSR_IA32_PLATFORM_ID_REGISTER PlatformIdMsr
;
572 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
573 CpuInfoInHob
[ProcessorNumber
].InitialApicId
= GetInitialApicId ();
574 CpuInfoInHob
[ProcessorNumber
].ApicId
= GetApicId ();
575 CpuInfoInHob
[ProcessorNumber
].Health
= BistData
;
576 CpuInfoInHob
[ProcessorNumber
].ApTopOfStack
= ApTopOfStack
;
578 CpuMpData
->CpuData
[ProcessorNumber
].Waiting
= FALSE
;
579 CpuMpData
->CpuData
[ProcessorNumber
].CpuHealthy
= (BistData
== 0) ? TRUE
: FALSE
;
582 // NOTE: PlatformId is not relevant on AMD platforms.
584 if (!StandardSignatureIsAuthenticAMD ()) {
585 PlatformIdMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_PLATFORM_ID
);
586 CpuMpData
->CpuData
[ProcessorNumber
].PlatformId
= (UINT8
)PlatformIdMsr
.Bits
.PlatformId
;
591 &CpuMpData
->CpuData
[ProcessorNumber
].ProcessorSignature
,
597 InitializeSpinLock(&CpuMpData
->CpuData
[ProcessorNumber
].ApLock
);
598 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateIdle
);
602 Get Protected mode code segment with 16-bit default addressing
603 from current GDT table.
605 @return Protected mode 16-bit code segment value.
609 GetProtectedMode16CS (
613 IA32_DESCRIPTOR GdtrDesc
;
614 IA32_SEGMENT_DESCRIPTOR
*GdtEntry
;
619 AsmReadGdtr (&GdtrDesc
);
620 GdtEntryCount
= (GdtrDesc
.Limit
+ 1) / sizeof (IA32_SEGMENT_DESCRIPTOR
);
621 GdtEntry
= (IA32_SEGMENT_DESCRIPTOR
*) GdtrDesc
.Base
;
622 for (Index
= 0; Index
< GdtEntryCount
; Index
++) {
623 if (GdtEntry
->Bits
.L
== 0 &&
624 GdtEntry
->Bits
.DB
== 0 &&
625 GdtEntry
->Bits
.Type
> 8) {
630 ASSERT (Index
!= GdtEntryCount
);
635 Get Protected mode code segment with 32-bit default addressing
636 from current GDT table.
638 @return Protected mode 32-bit code segment value.
642 GetProtectedMode32CS (
646 IA32_DESCRIPTOR GdtrDesc
;
647 IA32_SEGMENT_DESCRIPTOR
*GdtEntry
;
652 AsmReadGdtr (&GdtrDesc
);
653 GdtEntryCount
= (GdtrDesc
.Limit
+ 1) / sizeof (IA32_SEGMENT_DESCRIPTOR
);
654 GdtEntry
= (IA32_SEGMENT_DESCRIPTOR
*) GdtrDesc
.Base
;
655 for (Index
= 0; Index
< GdtEntryCount
; Index
++) {
656 if (GdtEntry
->Bits
.L
== 0 &&
657 GdtEntry
->Bits
.DB
== 1 &&
658 GdtEntry
->Bits
.Type
> 8) {
663 ASSERT (Index
!= GdtEntryCount
);
668 Reset an AP when in SEV-ES mode.
670 If successful, this function never returns.
672 @param[in] Ghcb Pointer to the GHCB
673 @param[in] CpuMpData Pointer to CPU MP Data
678 MpInitLibSevEsAPReset (
680 IN CPU_MP_DATA
*CpuMpData
683 UINT16 Code16
, Code32
;
688 Code16
= GetProtectedMode16CS ();
689 Code32
= GetProtectedMode32CS ();
691 if (CpuMpData
->WakeupBufferHigh
!= 0) {
692 APResetFn
= (AP_RESET
*) (CpuMpData
->WakeupBufferHigh
+ CpuMpData
->AddressMap
.SwitchToRealNoNxOffset
);
694 APResetFn
= (AP_RESET
*) (CpuMpData
->MpCpuExchangeInfo
->BufferStart
+ CpuMpData
->AddressMap
.SwitchToRealOffset
);
697 BufferStart
= CpuMpData
->MpCpuExchangeInfo
->BufferStart
;
698 StackStart
= CpuMpData
->SevEsAPResetStackStart
-
699 (AP_RESET_STACK_SIZE
* GetApicId ());
702 // This call never returns.
704 APResetFn (BufferStart
, Code16
, Code32
, StackStart
);
708 This function will be called from AP reset code if BSP uses WakeUpAP.
710 @param[in] ExchangeInfo Pointer to the MP exchange info buffer
711 @param[in] ApIndex Number of current executing AP
716 IN MP_CPU_EXCHANGE_INFO
*ExchangeInfo
,
720 CPU_MP_DATA
*CpuMpData
;
721 UINTN ProcessorNumber
;
722 EFI_AP_PROCEDURE Procedure
;
725 volatile UINT32
*ApStartupSignalBuffer
;
726 CPU_INFO_IN_HOB
*CpuInfoInHob
;
728 UINTN CurrentApicMode
;
731 // AP finished assembly code and begin to execute C code
733 CpuMpData
= ExchangeInfo
->CpuMpData
;
736 // AP's local APIC settings will be lost after received INIT IPI
737 // We need to re-initialize them at here
739 ProgramVirtualWireMode ();
741 // Mask the LINT0 and LINT1 so that AP doesn't enter the system timer interrupt handler.
743 DisableLvtInterrupts ();
744 SyncLocalApicTimerSetting (CpuMpData
);
746 CurrentApicMode
= GetApicMode ();
748 if (CpuMpData
->InitFlag
== ApInitConfig
) {
752 InterlockedIncrement ((UINT32
*) &CpuMpData
->CpuCount
);
753 ProcessorNumber
= ApIndex
;
755 // This is first time AP wakeup, get BIST information from AP stack
757 ApTopOfStack
= CpuMpData
->Buffer
+ (ProcessorNumber
+ 1) * CpuMpData
->CpuApStackSize
;
758 BistData
= *(UINT32
*) ((UINTN
) ApTopOfStack
- sizeof (UINTN
));
760 // CpuMpData->CpuData[0].VolatileRegisters is initialized based on BSP environment,
761 // to initialize AP in InitConfig path.
762 // NOTE: IDTR.BASE stored in CpuMpData->CpuData[0].VolatileRegisters points to a different IDT shared by all APs.
764 RestoreVolatileRegisters (&CpuMpData
->CpuData
[0].VolatileRegisters
, FALSE
);
765 InitializeApData (CpuMpData
, ProcessorNumber
, BistData
, ApTopOfStack
);
766 ApStartupSignalBuffer
= CpuMpData
->CpuData
[ProcessorNumber
].StartupApSignal
;
769 // Delay decrementing the APs executing count when SEV-ES is enabled
770 // to allow the APs to issue an AP_RESET_HOLD before the BSP possibly
771 // performs another INIT-SIPI-SIPI sequence.
773 if (!CpuMpData
->SevEsIsEnabled
) {
774 InterlockedDecrement ((UINT32
*) &CpuMpData
->MpCpuExchangeInfo
->NumApsExecuting
);
778 // Execute AP function if AP is ready
780 GetProcessorNumber (CpuMpData
, &ProcessorNumber
);
782 // Clear AP start-up signal when AP waken up
784 ApStartupSignalBuffer
= CpuMpData
->CpuData
[ProcessorNumber
].StartupApSignal
;
785 InterlockedCompareExchange32 (
786 (UINT32
*) ApStartupSignalBuffer
,
791 if (CpuMpData
->InitFlag
== ApInitReconfig
) {
793 // ApInitReconfig happens when:
794 // 1. AP is re-enabled after it's disabled, in either PEI or DXE phase.
795 // 2. AP is initialized in DXE phase.
796 // In either case, use the volatile registers value derived from BSP.
797 // NOTE: IDTR.BASE stored in CpuMpData->CpuData[0].VolatileRegisters points to a
798 // different IDT shared by all APs.
800 RestoreVolatileRegisters (&CpuMpData
->CpuData
[0].VolatileRegisters
, FALSE
);
802 if (CpuMpData
->ApLoopMode
== ApInHltLoop
) {
804 // Restore AP's volatile registers saved before AP is halted
806 RestoreVolatileRegisters (&CpuMpData
->CpuData
[ProcessorNumber
].VolatileRegisters
, TRUE
);
809 // The CPU driver might not flush TLB for APs on spot after updating
810 // page attributes. AP in mwait loop mode needs to take care of it when
817 if (GetApState (&CpuMpData
->CpuData
[ProcessorNumber
]) == CpuStateReady
) {
818 Procedure
= (EFI_AP_PROCEDURE
)CpuMpData
->CpuData
[ProcessorNumber
].ApFunction
;
819 Parameter
= (VOID
*) CpuMpData
->CpuData
[ProcessorNumber
].ApFunctionArgument
;
820 if (Procedure
!= NULL
) {
821 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateBusy
);
823 // Enable source debugging on AP function
827 // Invoke AP function here
829 Procedure (Parameter
);
830 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
831 if (CpuMpData
->SwitchBspFlag
) {
833 // Re-get the processor number due to BSP/AP maybe exchange in AP function
835 GetProcessorNumber (CpuMpData
, &ProcessorNumber
);
836 CpuMpData
->CpuData
[ProcessorNumber
].ApFunction
= 0;
837 CpuMpData
->CpuData
[ProcessorNumber
].ApFunctionArgument
= 0;
838 ApStartupSignalBuffer
= CpuMpData
->CpuData
[ProcessorNumber
].StartupApSignal
;
839 CpuInfoInHob
[ProcessorNumber
].ApTopOfStack
= CpuInfoInHob
[CpuMpData
->NewBspNumber
].ApTopOfStack
;
841 if (CpuInfoInHob
[ProcessorNumber
].ApicId
!= GetApicId () ||
842 CpuInfoInHob
[ProcessorNumber
].InitialApicId
!= GetInitialApicId ()) {
843 if (CurrentApicMode
!= GetApicMode ()) {
845 // If APIC mode change happened during AP function execution,
846 // we do not support APIC ID value changed.
852 // Re-get the CPU APICID and Initial APICID if they are changed
854 CpuInfoInHob
[ProcessorNumber
].ApicId
= GetApicId ();
855 CpuInfoInHob
[ProcessorNumber
].InitialApicId
= GetInitialApicId ();
860 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateFinished
);
865 // AP finished executing C code
867 InterlockedIncrement ((UINT32
*) &CpuMpData
->FinishedCount
);
870 // Place AP is specified loop mode
872 if (CpuMpData
->ApLoopMode
== ApInHltLoop
) {
874 // Save AP volatile registers
876 SaveVolatileRegisters (&CpuMpData
->CpuData
[ProcessorNumber
].VolatileRegisters
);
878 // Place AP in HLT-loop
881 DisableInterrupts ();
882 if (CpuMpData
->SevEsIsEnabled
) {
883 MSR_SEV_ES_GHCB_REGISTER Msr
;
888 DoDecrement
= (BOOLEAN
) (CpuMpData
->InitFlag
== ApInitConfig
);
891 Msr
.GhcbPhysicalAddress
= AsmReadMsr64 (MSR_SEV_ES_GHCB
);
900 // Perform the delayed decrement just before issuing the first
901 // VMGEXIT with AP_RESET_HOLD.
903 InterlockedDecrement ((UINT32
*) &CpuMpData
->MpCpuExchangeInfo
->NumApsExecuting
);
906 Status
= VmgExit (Ghcb
, SVM_EXIT_AP_RESET_HOLD
, 0, 0);
907 if ((Status
== 0) && (Ghcb
->SaveArea
.SwExitInfo2
!= 0)) {
916 // Awakened in a new phase? Use the new CpuMpData
918 if (CpuMpData
->NewCpuMpData
!= NULL
) {
919 CpuMpData
= CpuMpData
->NewCpuMpData
;
922 MpInitLibSevEsAPReset (Ghcb
, CpuMpData
);
930 DisableInterrupts ();
931 if (CpuMpData
->ApLoopMode
== ApInMwaitLoop
) {
933 // Place AP in MWAIT-loop
935 AsmMonitor ((UINTN
) ApStartupSignalBuffer
, 0, 0);
936 if (*ApStartupSignalBuffer
!= WAKEUP_AP_SIGNAL
) {
938 // Check AP start-up signal again.
939 // If AP start-up signal is not set, place AP into
940 // the specified C-state
942 AsmMwait (CpuMpData
->ApTargetCState
<< 4, 0);
944 } else if (CpuMpData
->ApLoopMode
== ApInRunLoop
) {
946 // Place AP in Run-loop
954 // If AP start-up signal is written, AP is waken up
955 // otherwise place AP in loop again
957 if (*ApStartupSignalBuffer
== WAKEUP_AP_SIGNAL
) {
965 Wait for AP wakeup and write AP start-up signal till AP is waken up.
967 @param[in] ApStartupSignalBuffer Pointer to AP wakeup signal
971 IN
volatile UINT32
*ApStartupSignalBuffer
975 // If AP is waken up, StartupApSignal should be cleared.
976 // Otherwise, write StartupApSignal again till AP waken up.
978 while (InterlockedCompareExchange32 (
979 (UINT32
*) ApStartupSignalBuffer
,
988 This function will fill the exchange info structure.
990 @param[in] CpuMpData Pointer to CPU MP Data
994 FillExchangeInfoData (
995 IN CPU_MP_DATA
*CpuMpData
998 volatile MP_CPU_EXCHANGE_INFO
*ExchangeInfo
;
1000 IA32_SEGMENT_DESCRIPTOR
*Selector
;
1003 ExchangeInfo
= CpuMpData
->MpCpuExchangeInfo
;
1004 ExchangeInfo
->Lock
= 0;
1005 ExchangeInfo
->StackStart
= CpuMpData
->Buffer
;
1006 ExchangeInfo
->StackSize
= CpuMpData
->CpuApStackSize
;
1007 ExchangeInfo
->BufferStart
= CpuMpData
->WakeupBuffer
;
1008 ExchangeInfo
->ModeOffset
= CpuMpData
->AddressMap
.ModeEntryOffset
;
1010 ExchangeInfo
->CodeSegment
= AsmReadCs ();
1011 ExchangeInfo
->DataSegment
= AsmReadDs ();
1013 ExchangeInfo
->Cr3
= AsmReadCr3 ();
1015 ExchangeInfo
->CFunction
= (UINTN
) ApWakeupFunction
;
1016 ExchangeInfo
->ApIndex
= 0;
1017 ExchangeInfo
->NumApsExecuting
= 0;
1018 ExchangeInfo
->InitFlag
= (UINTN
) CpuMpData
->InitFlag
;
1019 ExchangeInfo
->CpuInfo
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
1020 ExchangeInfo
->CpuMpData
= CpuMpData
;
1022 ExchangeInfo
->EnableExecuteDisable
= IsBspExecuteDisableEnabled ();
1024 ExchangeInfo
->InitializeFloatingPointUnitsAddress
= (UINTN
)InitializeFloatingPointUnits
;
1027 // We can check either CPUID(7).ECX[bit16] or check CR4.LA57[bit12]
1028 // to determin whether 5-Level Paging is enabled.
1029 // CPUID(7).ECX[bit16] shows CPU's capability, CR4.LA57[bit12] shows
1030 // current system setting.
1031 // Using latter way is simpler because it also eliminates the needs to
1032 // check whether platform wants to enable it.
1034 Cr4
.UintN
= AsmReadCr4 ();
1035 ExchangeInfo
->Enable5LevelPaging
= (BOOLEAN
) (Cr4
.Bits
.LA57
== 1);
1036 DEBUG ((DEBUG_INFO
, "%a: 5-Level Paging = %d\n", gEfiCallerBaseName
, ExchangeInfo
->Enable5LevelPaging
));
1038 ExchangeInfo
->SevEsIsEnabled
= CpuMpData
->SevEsIsEnabled
;
1039 ExchangeInfo
->GhcbBase
= (UINTN
) CpuMpData
->GhcbBase
;
1042 // Get the BSP's data of GDT and IDT
1044 AsmReadGdtr ((IA32_DESCRIPTOR
*) &ExchangeInfo
->GdtrProfile
);
1045 AsmReadIdtr ((IA32_DESCRIPTOR
*) &ExchangeInfo
->IdtrProfile
);
1048 // Find a 32-bit code segment
1050 Selector
= (IA32_SEGMENT_DESCRIPTOR
*)ExchangeInfo
->GdtrProfile
.Base
;
1051 Size
= ExchangeInfo
->GdtrProfile
.Limit
+ 1;
1053 if (Selector
->Bits
.L
== 0 && Selector
->Bits
.Type
>= 8) {
1054 ExchangeInfo
->ModeTransitionSegment
=
1055 (UINT16
)((UINTN
)Selector
- ExchangeInfo
->GdtrProfile
.Base
);
1059 Size
-= sizeof (IA32_SEGMENT_DESCRIPTOR
);
1063 // Copy all 32-bit code and 64-bit code into memory with type of
1064 // EfiBootServicesCode to avoid page fault if NX memory protection is enabled.
1066 if (CpuMpData
->WakeupBufferHigh
!= 0) {
1067 Size
= CpuMpData
->AddressMap
.RendezvousFunnelSize
+
1068 CpuMpData
->AddressMap
.SwitchToRealSize
-
1069 CpuMpData
->AddressMap
.ModeTransitionOffset
;
1071 (VOID
*)CpuMpData
->WakeupBufferHigh
,
1072 CpuMpData
->AddressMap
.RendezvousFunnelAddress
+
1073 CpuMpData
->AddressMap
.ModeTransitionOffset
,
1077 ExchangeInfo
->ModeTransitionMemory
= (UINT32
)CpuMpData
->WakeupBufferHigh
;
1079 ExchangeInfo
->ModeTransitionMemory
= (UINT32
)
1080 (ExchangeInfo
->BufferStart
+ CpuMpData
->AddressMap
.ModeTransitionOffset
);
1083 ExchangeInfo
->ModeHighMemory
= ExchangeInfo
->ModeTransitionMemory
+
1084 (UINT32
)ExchangeInfo
->ModeOffset
-
1085 (UINT32
)CpuMpData
->AddressMap
.ModeTransitionOffset
;
1086 ExchangeInfo
->ModeHighSegment
= (UINT16
)ExchangeInfo
->CodeSegment
;
1090 Helper function that waits until the finished AP count reaches the specified
1091 limit, or the specified timeout elapses (whichever comes first).
1093 @param[in] CpuMpData Pointer to CPU MP Data.
1094 @param[in] FinishedApLimit The number of finished APs to wait for.
1095 @param[in] TimeLimit The number of microseconds to wait for.
1098 TimedWaitForApFinish (
1099 IN CPU_MP_DATA
*CpuMpData
,
1100 IN UINT32 FinishedApLimit
,
1105 Get available system memory below 1MB by specified size.
1107 @param[in] CpuMpData The pointer to CPU MP Data structure.
1110 BackupAndPrepareWakeupBuffer(
1111 IN CPU_MP_DATA
*CpuMpData
1115 (VOID
*) CpuMpData
->BackupBuffer
,
1116 (VOID
*) CpuMpData
->WakeupBuffer
,
1117 CpuMpData
->BackupBufferSize
1120 (VOID
*) CpuMpData
->WakeupBuffer
,
1121 (VOID
*) CpuMpData
->AddressMap
.RendezvousFunnelAddress
,
1122 CpuMpData
->AddressMap
.RendezvousFunnelSize
+
1123 CpuMpData
->AddressMap
.SwitchToRealSize
1128 Restore wakeup buffer data.
1130 @param[in] CpuMpData The pointer to CPU MP Data structure.
1133 RestoreWakeupBuffer(
1134 IN CPU_MP_DATA
*CpuMpData
1138 (VOID
*) CpuMpData
->WakeupBuffer
,
1139 (VOID
*) CpuMpData
->BackupBuffer
,
1140 CpuMpData
->BackupBufferSize
1145 Calculate the size of the reset stack.
1147 @return Total amount of memory required for stacks
1151 GetApResetStackSize (
1155 return AP_RESET_STACK_SIZE
* PcdGet32(PcdCpuMaxLogicalProcessorNumber
);
1159 Calculate the size of the reset vector.
1161 @param[in] AddressMap The pointer to Address Map structure.
1163 @return Total amount of memory required for the AP reset area
1167 GetApResetVectorSize (
1168 IN MP_ASSEMBLY_ADDRESS_MAP
*AddressMap
1173 Size
= ALIGN_VALUE (AddressMap
->RendezvousFunnelSize
+
1174 AddressMap
->SwitchToRealSize
+
1175 sizeof (MP_CPU_EXCHANGE_INFO
),
1176 CPU_STACK_ALIGNMENT
);
1177 Size
+= GetApResetStackSize ();
1183 Allocate reset vector buffer.
1185 @param[in, out] CpuMpData The pointer to CPU MP Data structure.
1188 AllocateResetVector (
1189 IN OUT CPU_MP_DATA
*CpuMpData
1192 UINTN ApResetVectorSize
;
1194 if (CpuMpData
->WakeupBuffer
== (UINTN
) -1) {
1195 ApResetVectorSize
= GetApResetVectorSize (&CpuMpData
->AddressMap
);
1197 CpuMpData
->WakeupBuffer
= GetWakeupBuffer (ApResetVectorSize
);
1198 CpuMpData
->MpCpuExchangeInfo
= (MP_CPU_EXCHANGE_INFO
*) (UINTN
)
1199 (CpuMpData
->WakeupBuffer
+
1200 CpuMpData
->AddressMap
.RendezvousFunnelSize
+
1201 CpuMpData
->AddressMap
.SwitchToRealSize
);
1202 CpuMpData
->WakeupBufferHigh
= GetModeTransitionBuffer (
1203 CpuMpData
->AddressMap
.RendezvousFunnelSize
+
1204 CpuMpData
->AddressMap
.SwitchToRealSize
-
1205 CpuMpData
->AddressMap
.ModeTransitionOffset
1208 // The reset stack starts at the end of the buffer.
1210 CpuMpData
->SevEsAPResetStackStart
= CpuMpData
->WakeupBuffer
+ ApResetVectorSize
;
1212 BackupAndPrepareWakeupBuffer (CpuMpData
);
1216 Free AP reset vector buffer.
1218 @param[in] CpuMpData The pointer to CPU MP Data structure.
1222 IN CPU_MP_DATA
*CpuMpData
1226 // If SEV-ES is enabled, the reset area is needed for AP parking and
1227 // and AP startup in the OS, so the reset area is reserved. Do not
1228 // perform the restore as this will overwrite memory which has data
1229 // needed by SEV-ES.
1231 if (!CpuMpData
->SevEsIsEnabled
) {
1232 RestoreWakeupBuffer (CpuMpData
);
1237 Allocate the SEV-ES AP jump table buffer.
1239 @param[in, out] CpuMpData The pointer to CPU MP Data structure.
1242 AllocateSevEsAPMemory (
1243 IN OUT CPU_MP_DATA
*CpuMpData
1246 if (CpuMpData
->SevEsAPBuffer
== (UINTN
) -1) {
1247 CpuMpData
->SevEsAPBuffer
=
1248 CpuMpData
->SevEsIsEnabled
? GetSevEsAPMemory () : 0;
1253 Program the SEV-ES AP jump table buffer.
1255 @param[in] SipiVector The SIPI vector used for the AP Reset
1262 SEV_ES_AP_JMP_FAR
*JmpFar
;
1263 UINT32 Offset
, InsnByte
;
1266 JmpFar
= (SEV_ES_AP_JMP_FAR
*) FixedPcdGet32 (PcdSevEsWorkAreaBase
);
1267 ASSERT (JmpFar
!= NULL
);
1270 // Obtain the address of the Segment/Rip location in the workarea.
1271 // This will be set to a value derived from the SIPI vector and will
1272 // be the memory address used for the far jump below.
1274 Offset
= FixedPcdGet32 (PcdSevEsWorkAreaBase
);
1275 Offset
+= sizeof (JmpFar
->InsnBuffer
);
1276 LoNib
= (UINT8
) Offset
;
1277 HiNib
= (UINT8
) (Offset
>> 8);
1280 // Program the workarea (which is the initial AP boot address) with
1281 // far jump to the SIPI vector (where XX and YY represent the
1282 // address of where the SIPI vector is stored.
1284 // JMP FAR [CS:XXYY] => 2E FF 2E YY XX
1287 JmpFar
->InsnBuffer
[InsnByte
++] = 0x2E; // CS override prefix
1288 JmpFar
->InsnBuffer
[InsnByte
++] = 0xFF; // JMP (FAR)
1289 JmpFar
->InsnBuffer
[InsnByte
++] = 0x2E; // ModRM (JMP memory location)
1290 JmpFar
->InsnBuffer
[InsnByte
++] = LoNib
; // YY offset ...
1291 JmpFar
->InsnBuffer
[InsnByte
++] = HiNib
; // XX offset ...
1294 // Program the Segment/Rip based on the SIPI vector (always at least
1295 // 16-byte aligned, so Rip is set to 0).
1298 JmpFar
->Segment
= (UINT16
) (SipiVector
>> 4);
1302 This function will be called by BSP to wakeup AP.
1304 @param[in] CpuMpData Pointer to CPU MP Data
1305 @param[in] Broadcast TRUE: Send broadcast IPI to all APs
1306 FALSE: Send IPI to AP by ApicId
1307 @param[in] ProcessorNumber The handle number of specified processor
1308 @param[in] Procedure The function to be invoked by AP
1309 @param[in] ProcedureArgument The argument to be passed into AP function
1310 @param[in] WakeUpDisabledAps Whether need to wake up disabled APs in broadcast mode.
1314 IN CPU_MP_DATA
*CpuMpData
,
1315 IN BOOLEAN Broadcast
,
1316 IN UINTN ProcessorNumber
,
1317 IN EFI_AP_PROCEDURE Procedure
, OPTIONAL
1318 IN VOID
*ProcedureArgument
, OPTIONAL
1319 IN BOOLEAN WakeUpDisabledAps
1322 volatile MP_CPU_EXCHANGE_INFO
*ExchangeInfo
;
1324 CPU_AP_DATA
*CpuData
;
1325 BOOLEAN ResetVectorRequired
;
1326 CPU_INFO_IN_HOB
*CpuInfoInHob
;
1328 CpuMpData
->FinishedCount
= 0;
1329 ResetVectorRequired
= FALSE
;
1331 if (CpuMpData
->WakeUpByInitSipiSipi
||
1332 CpuMpData
->InitFlag
!= ApInitDone
) {
1333 ResetVectorRequired
= TRUE
;
1334 AllocateResetVector (CpuMpData
);
1335 AllocateSevEsAPMemory (CpuMpData
);
1336 FillExchangeInfoData (CpuMpData
);
1337 SaveLocalApicTimerSetting (CpuMpData
);
1340 if (CpuMpData
->ApLoopMode
== ApInMwaitLoop
) {
1342 // Get AP target C-state each time when waking up AP,
1343 // for it maybe updated by platform again
1345 CpuMpData
->ApTargetCState
= PcdGet8 (PcdCpuApTargetCstate
);
1348 ExchangeInfo
= CpuMpData
->MpCpuExchangeInfo
;
1351 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
1352 if (Index
!= CpuMpData
->BspNumber
) {
1353 CpuData
= &CpuMpData
->CpuData
[Index
];
1355 // All AP(include disabled AP) will be woke up by INIT-SIPI-SIPI, but
1356 // the AP procedure will be skipped for disabled AP because AP state
1357 // is not CpuStateReady.
1359 if (GetApState (CpuData
) == CpuStateDisabled
&& !WakeUpDisabledAps
) {
1363 CpuData
->ApFunction
= (UINTN
) Procedure
;
1364 CpuData
->ApFunctionArgument
= (UINTN
) ProcedureArgument
;
1365 SetApState (CpuData
, CpuStateReady
);
1366 if (CpuMpData
->InitFlag
!= ApInitConfig
) {
1367 *(UINT32
*) CpuData
->StartupApSignal
= WAKEUP_AP_SIGNAL
;
1371 if (ResetVectorRequired
) {
1373 // For SEV-ES, the initial AP boot address will be defined by
1374 // PcdSevEsWorkAreaBase. The Segment/Rip must be the jump address
1375 // from the original INIT-SIPI-SIPI.
1377 if (CpuMpData
->SevEsIsEnabled
) {
1378 SetSevEsJumpTable (ExchangeInfo
->BufferStart
);
1384 SendInitSipiSipiAllExcludingSelf ((UINT32
) ExchangeInfo
->BufferStart
);
1386 if (CpuMpData
->InitFlag
== ApInitConfig
) {
1387 if (PcdGet32 (PcdCpuBootLogicalProcessorNumber
) > 0) {
1389 // The AP enumeration algorithm below is suitable only when the
1390 // platform can tell us the *exact* boot CPU count in advance.
1392 // The wait below finishes only when the detected AP count reaches
1393 // (PcdCpuBootLogicalProcessorNumber - 1), regardless of how long that
1394 // takes. If at least one AP fails to check in (meaning a platform
1395 // hardware bug), the detection hangs forever, by design. If the actual
1396 // boot CPU count in the system is higher than
1397 // PcdCpuBootLogicalProcessorNumber (meaning a platform
1398 // misconfiguration), then some APs may complete initialization after
1399 // the wait finishes, and cause undefined behavior.
1401 TimedWaitForApFinish (
1403 PcdGet32 (PcdCpuBootLogicalProcessorNumber
) - 1,
1404 MAX_UINT32
// approx. 71 minutes
1408 // The AP enumeration algorithm below is suitable for two use cases.
1410 // (1) The check-in time for an individual AP is bounded, and APs run
1411 // through their initialization routines strongly concurrently. In
1412 // particular, the number of concurrently running APs
1413 // ("NumApsExecuting") is never expected to fall to zero
1414 // *temporarily* -- it is expected to fall to zero only when all
1415 // APs have checked-in.
1417 // In this case, the platform is supposed to set
1418 // PcdCpuApInitTimeOutInMicroSeconds to a low-ish value (just long
1419 // enough for one AP to start initialization). The timeout will be
1420 // reached soon, and remaining APs are collected by watching
1421 // NumApsExecuting fall to zero. If NumApsExecuting falls to zero
1422 // mid-process, while some APs have not completed initialization,
1423 // the behavior is undefined.
1425 // (2) The check-in time for an individual AP is unbounded, and/or APs
1426 // may complete their initializations widely spread out. In
1427 // particular, some APs may finish initialization before some APs
1430 // In this case, the platform is supposed to set
1431 // PcdCpuApInitTimeOutInMicroSeconds to a high-ish value. The AP
1432 // enumeration will always take that long (except when the boot CPU
1433 // count happens to be maximal, that is,
1434 // PcdCpuMaxLogicalProcessorNumber). All APs are expected to
1435 // check-in before the timeout, and NumApsExecuting is assumed zero
1436 // at timeout. APs that miss the time-out may cause undefined
1439 TimedWaitForApFinish (
1441 PcdGet32 (PcdCpuMaxLogicalProcessorNumber
) - 1,
1442 PcdGet32 (PcdCpuApInitTimeOutInMicroSeconds
)
1445 while (CpuMpData
->MpCpuExchangeInfo
->NumApsExecuting
!= 0) {
1451 // Wait all APs waken up if this is not the 1st broadcast of SIPI
1453 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
1454 CpuData
= &CpuMpData
->CpuData
[Index
];
1455 if (Index
!= CpuMpData
->BspNumber
) {
1456 WaitApWakeup (CpuData
->StartupApSignal
);
1461 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
1462 CpuData
->ApFunction
= (UINTN
) Procedure
;
1463 CpuData
->ApFunctionArgument
= (UINTN
) ProcedureArgument
;
1464 SetApState (CpuData
, CpuStateReady
);
1466 // Wakeup specified AP
1468 ASSERT (CpuMpData
->InitFlag
!= ApInitConfig
);
1469 *(UINT32
*) CpuData
->StartupApSignal
= WAKEUP_AP_SIGNAL
;
1470 if (ResetVectorRequired
) {
1471 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
1474 // For SEV-ES, the initial AP boot address will be defined by
1475 // PcdSevEsWorkAreaBase. The Segment/Rip must be the jump address
1476 // from the original INIT-SIPI-SIPI.
1478 if (CpuMpData
->SevEsIsEnabled
) {
1479 SetSevEsJumpTable (ExchangeInfo
->BufferStart
);
1483 CpuInfoInHob
[ProcessorNumber
].ApicId
,
1484 (UINT32
) ExchangeInfo
->BufferStart
1488 // Wait specified AP waken up
1490 WaitApWakeup (CpuData
->StartupApSignal
);
1493 if (ResetVectorRequired
) {
1494 FreeResetVector (CpuMpData
);
1498 // After one round of Wakeup Ap actions, need to re-sync ApLoopMode with
1499 // WakeUpByInitSipiSipi flag. WakeUpByInitSipiSipi flag maybe changed by
1500 // S3SmmInitDone Ppi.
1502 CpuMpData
->WakeUpByInitSipiSipi
= (CpuMpData
->ApLoopMode
== ApInHltLoop
);
1506 Calculate timeout value and return the current performance counter value.
1508 Calculate the number of performance counter ticks required for a timeout.
1509 If TimeoutInMicroseconds is 0, return value is also 0, which is recognized
1512 @param[in] TimeoutInMicroseconds Timeout value in microseconds.
1513 @param[out] CurrentTime Returns the current value of the performance counter.
1515 @return Expected time stamp counter for timeout.
1516 If TimeoutInMicroseconds is 0, return value is also 0, which is recognized
1522 IN UINTN TimeoutInMicroseconds
,
1523 OUT UINT64
*CurrentTime
1526 UINT64 TimeoutInSeconds
;
1527 UINT64 TimestampCounterFreq
;
1530 // Read the current value of the performance counter
1532 *CurrentTime
= GetPerformanceCounter ();
1535 // If TimeoutInMicroseconds is 0, return value is also 0, which is recognized
1538 if (TimeoutInMicroseconds
== 0) {
1543 // GetPerformanceCounterProperties () returns the timestamp counter's frequency
1546 TimestampCounterFreq
= GetPerformanceCounterProperties (NULL
, NULL
);
1549 // Check the potential overflow before calculate the number of ticks for the timeout value.
1551 if (DivU64x64Remainder (MAX_UINT64
, TimeoutInMicroseconds
, NULL
) < TimestampCounterFreq
) {
1553 // Convert microseconds into seconds if direct multiplication overflows
1555 TimeoutInSeconds
= DivU64x32 (TimeoutInMicroseconds
, 1000000);
1557 // Assertion if the final tick count exceeds MAX_UINT64
1559 ASSERT (DivU64x64Remainder (MAX_UINT64
, TimeoutInSeconds
, NULL
) >= TimestampCounterFreq
);
1560 return MultU64x64 (TimestampCounterFreq
, TimeoutInSeconds
);
1563 // No overflow case, multiply the return value with TimeoutInMicroseconds and then divide
1564 // it by 1,000,000, to get the number of ticks for the timeout value.
1568 TimestampCounterFreq
,
1569 TimeoutInMicroseconds
1577 Checks whether timeout expires.
1579 Check whether the number of elapsed performance counter ticks required for
1580 a timeout condition has been reached.
1581 If Timeout is zero, which means infinity, return value is always FALSE.
1583 @param[in, out] PreviousTime On input, the value of the performance counter
1584 when it was last read.
1585 On output, the current value of the performance
1587 @param[in] TotalTime The total amount of elapsed time in performance
1589 @param[in] Timeout The number of performance counter ticks required
1590 to reach a timeout condition.
1592 @retval TRUE A timeout condition has been reached.
1593 @retval FALSE A timeout condition has not been reached.
1598 IN OUT UINT64
*PreviousTime
,
1599 IN UINT64
*TotalTime
,
1612 GetPerformanceCounterProperties (&Start
, &End
);
1613 Cycle
= End
- Start
;
1618 CurrentTime
= GetPerformanceCounter();
1619 Delta
= (INT64
) (CurrentTime
- *PreviousTime
);
1626 *TotalTime
+= Delta
;
1627 *PreviousTime
= CurrentTime
;
1628 if (*TotalTime
> Timeout
) {
1635 Helper function that waits until the finished AP count reaches the specified
1636 limit, or the specified timeout elapses (whichever comes first).
1638 @param[in] CpuMpData Pointer to CPU MP Data.
1639 @param[in] FinishedApLimit The number of finished APs to wait for.
1640 @param[in] TimeLimit The number of microseconds to wait for.
1643 TimedWaitForApFinish (
1644 IN CPU_MP_DATA
*CpuMpData
,
1645 IN UINT32 FinishedApLimit
,
1650 // CalculateTimeout() and CheckTimeout() consider a TimeLimit of 0
1651 // "infinity", so check for (TimeLimit == 0) explicitly.
1653 if (TimeLimit
== 0) {
1657 CpuMpData
->TotalTime
= 0;
1658 CpuMpData
->ExpectedTime
= CalculateTimeout (
1660 &CpuMpData
->CurrentTime
1662 while (CpuMpData
->FinishedCount
< FinishedApLimit
&&
1664 &CpuMpData
->CurrentTime
,
1665 &CpuMpData
->TotalTime
,
1666 CpuMpData
->ExpectedTime
1671 if (CpuMpData
->FinishedCount
>= FinishedApLimit
) {
1674 "%a: reached FinishedApLimit=%u in %Lu microseconds\n",
1677 DivU64x64Remainder (
1678 MultU64x32 (CpuMpData
->TotalTime
, 1000000),
1679 GetPerformanceCounterProperties (NULL
, NULL
),
1687 Reset an AP to Idle state.
1689 Any task being executed by the AP will be aborted and the AP
1690 will be waiting for a new task in Wait-For-SIPI state.
1692 @param[in] ProcessorNumber The handle number of processor.
1695 ResetProcessorToIdleState (
1696 IN UINTN ProcessorNumber
1699 CPU_MP_DATA
*CpuMpData
;
1701 CpuMpData
= GetCpuMpData ();
1703 CpuMpData
->InitFlag
= ApInitReconfig
;
1704 WakeUpAP (CpuMpData
, FALSE
, ProcessorNumber
, NULL
, NULL
, TRUE
);
1705 while (CpuMpData
->FinishedCount
< 1) {
1708 CpuMpData
->InitFlag
= ApInitDone
;
1710 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateIdle
);
1714 Searches for the next waiting AP.
1716 Search for the next AP that is put in waiting state by single-threaded StartupAllAPs().
1718 @param[out] NextProcessorNumber Pointer to the processor number of the next waiting AP.
1720 @retval EFI_SUCCESS The next waiting AP has been found.
1721 @retval EFI_NOT_FOUND No waiting AP exists.
1725 GetNextWaitingProcessorNumber (
1726 OUT UINTN
*NextProcessorNumber
1729 UINTN ProcessorNumber
;
1730 CPU_MP_DATA
*CpuMpData
;
1732 CpuMpData
= GetCpuMpData ();
1734 for (ProcessorNumber
= 0; ProcessorNumber
< CpuMpData
->CpuCount
; ProcessorNumber
++) {
1735 if (CpuMpData
->CpuData
[ProcessorNumber
].Waiting
) {
1736 *NextProcessorNumber
= ProcessorNumber
;
1741 return EFI_NOT_FOUND
;
1744 /** Checks status of specified AP.
1746 This function checks whether the specified AP has finished the task assigned
1747 by StartupThisAP(), and whether timeout expires.
1749 @param[in] ProcessorNumber The handle number of processor.
1751 @retval EFI_SUCCESS Specified AP has finished task assigned by StartupThisAPs().
1752 @retval EFI_TIMEOUT The timeout expires.
1753 @retval EFI_NOT_READY Specified AP has not finished task and timeout has not expired.
1757 IN UINTN ProcessorNumber
1760 CPU_MP_DATA
*CpuMpData
;
1761 CPU_AP_DATA
*CpuData
;
1763 CpuMpData
= GetCpuMpData ();
1764 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
1767 // Check the CPU state of AP. If it is CpuStateIdle, then the AP has finished its task.
1768 // Only BSP and corresponding AP access this unit of CPU Data. This means the AP will not modify the
1769 // value of state after setting the it to CpuStateIdle, so BSP can safely make use of its value.
1772 // If the AP finishes for StartupThisAP(), return EFI_SUCCESS.
1774 if (GetApState(CpuData
) == CpuStateFinished
) {
1775 if (CpuData
->Finished
!= NULL
) {
1776 *(CpuData
->Finished
) = TRUE
;
1778 SetApState (CpuData
, CpuStateIdle
);
1782 // If timeout expires for StartupThisAP(), report timeout.
1784 if (CheckTimeout (&CpuData
->CurrentTime
, &CpuData
->TotalTime
, CpuData
->ExpectedTime
)) {
1785 if (CpuData
->Finished
!= NULL
) {
1786 *(CpuData
->Finished
) = FALSE
;
1789 // Reset failed AP to idle state
1791 ResetProcessorToIdleState (ProcessorNumber
);
1796 return EFI_NOT_READY
;
1800 Checks status of all APs.
1802 This function checks whether all APs have finished task assigned by StartupAllAPs(),
1803 and whether timeout expires.
1805 @retval EFI_SUCCESS All APs have finished task assigned by StartupAllAPs().
1806 @retval EFI_TIMEOUT The timeout expires.
1807 @retval EFI_NOT_READY APs have not finished task and timeout has not expired.
1814 UINTN ProcessorNumber
;
1815 UINTN NextProcessorNumber
;
1818 CPU_MP_DATA
*CpuMpData
;
1819 CPU_AP_DATA
*CpuData
;
1821 CpuMpData
= GetCpuMpData ();
1823 NextProcessorNumber
= 0;
1826 // Go through all APs that are responsible for the StartupAllAPs().
1828 for (ProcessorNumber
= 0; ProcessorNumber
< CpuMpData
->CpuCount
; ProcessorNumber
++) {
1829 if (!CpuMpData
->CpuData
[ProcessorNumber
].Waiting
) {
1833 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
1835 // Check the CPU state of AP. If it is CpuStateIdle, then the AP has finished its task.
1836 // Only BSP and corresponding AP access this unit of CPU Data. This means the AP will not modify the
1837 // value of state after setting the it to CpuStateIdle, so BSP can safely make use of its value.
1839 if (GetApState(CpuData
) == CpuStateFinished
) {
1840 CpuMpData
->RunningCount
--;
1841 CpuMpData
->CpuData
[ProcessorNumber
].Waiting
= FALSE
;
1842 SetApState(CpuData
, CpuStateIdle
);
1845 // If in Single Thread mode, then search for the next waiting AP for execution.
1847 if (CpuMpData
->SingleThread
) {
1848 Status
= GetNextWaitingProcessorNumber (&NextProcessorNumber
);
1850 if (!EFI_ERROR (Status
)) {
1854 (UINT32
) NextProcessorNumber
,
1855 CpuMpData
->Procedure
,
1856 CpuMpData
->ProcArguments
,
1865 // If all APs finish, return EFI_SUCCESS.
1867 if (CpuMpData
->RunningCount
== 0) {
1872 // If timeout expires, report timeout.
1875 &CpuMpData
->CurrentTime
,
1876 &CpuMpData
->TotalTime
,
1877 CpuMpData
->ExpectedTime
)
1880 // If FailedCpuList is not NULL, record all failed APs in it.
1882 if (CpuMpData
->FailedCpuList
!= NULL
) {
1883 *CpuMpData
->FailedCpuList
=
1884 AllocatePool ((CpuMpData
->RunningCount
+ 1) * sizeof (UINTN
));
1885 ASSERT (*CpuMpData
->FailedCpuList
!= NULL
);
1889 for (ProcessorNumber
= 0; ProcessorNumber
< CpuMpData
->CpuCount
; ProcessorNumber
++) {
1891 // Check whether this processor is responsible for StartupAllAPs().
1893 if (CpuMpData
->CpuData
[ProcessorNumber
].Waiting
) {
1895 // Reset failed APs to idle state
1897 ResetProcessorToIdleState (ProcessorNumber
);
1898 CpuMpData
->CpuData
[ProcessorNumber
].Waiting
= FALSE
;
1899 if (CpuMpData
->FailedCpuList
!= NULL
) {
1900 (*CpuMpData
->FailedCpuList
)[ListIndex
++] = ProcessorNumber
;
1904 if (CpuMpData
->FailedCpuList
!= NULL
) {
1905 (*CpuMpData
->FailedCpuList
)[ListIndex
] = END_OF_CPU_LIST
;
1909 return EFI_NOT_READY
;
1913 MP Initialize Library initialization.
1915 This service will allocate AP reset vector and wakeup all APs to do APs
1918 This service must be invoked before all other MP Initialize Library
1919 service are invoked.
1921 @retval EFI_SUCCESS MP initialization succeeds.
1922 @retval Others MP initialization fails.
1927 MpInitLibInitialize (
1931 CPU_MP_DATA
*OldCpuMpData
;
1932 CPU_INFO_IN_HOB
*CpuInfoInHob
;
1933 UINT32 MaxLogicalProcessorNumber
;
1935 MP_ASSEMBLY_ADDRESS_MAP AddressMap
;
1936 CPU_VOLATILE_REGISTERS VolatileRegisters
;
1938 UINT32 MonitorFilterSize
;
1941 CPU_MP_DATA
*CpuMpData
;
1943 UINT8
*MonitorBuffer
;
1945 UINTN ApResetVectorSize
;
1946 UINTN BackupBufferAddr
;
1949 OldCpuMpData
= GetCpuMpDataFromGuidedHob ();
1950 if (OldCpuMpData
== NULL
) {
1951 MaxLogicalProcessorNumber
= PcdGet32(PcdCpuMaxLogicalProcessorNumber
);
1953 MaxLogicalProcessorNumber
= OldCpuMpData
->CpuCount
;
1955 ASSERT (MaxLogicalProcessorNumber
!= 0);
1957 AsmGetAddressMap (&AddressMap
);
1958 ApResetVectorSize
= GetApResetVectorSize (&AddressMap
);
1959 ApStackSize
= PcdGet32(PcdCpuApStackSize
);
1960 ApLoopMode
= GetApLoopMode (&MonitorFilterSize
);
1963 // Save BSP's Control registers for APs.
1965 SaveVolatileRegisters (&VolatileRegisters
);
1967 BufferSize
= ApStackSize
* MaxLogicalProcessorNumber
;
1968 BufferSize
+= MonitorFilterSize
* MaxLogicalProcessorNumber
;
1969 BufferSize
+= ApResetVectorSize
;
1970 BufferSize
= ALIGN_VALUE (BufferSize
, 8);
1971 BufferSize
+= VolatileRegisters
.Idtr
.Limit
+ 1;
1972 BufferSize
+= sizeof (CPU_MP_DATA
);
1973 BufferSize
+= (sizeof (CPU_AP_DATA
) + sizeof (CPU_INFO_IN_HOB
))* MaxLogicalProcessorNumber
;
1974 MpBuffer
= AllocatePages (EFI_SIZE_TO_PAGES (BufferSize
));
1975 ASSERT (MpBuffer
!= NULL
);
1976 ZeroMem (MpBuffer
, BufferSize
);
1977 Buffer
= (UINTN
) MpBuffer
;
1980 // The layout of the Buffer is as below:
1982 // +--------------------+ <-- Buffer
1984 // +--------------------+ <-- MonitorBuffer
1985 // AP Monitor Filters (N)
1986 // +--------------------+ <-- BackupBufferAddr (CpuMpData->BackupBuffer)
1988 // +--------------------+
1990 // +--------------------+ <-- ApIdtBase (8-byte boundary)
1991 // AP IDT All APs share one separate IDT. So AP can get address of CPU_MP_DATA from IDT Base.
1992 // +--------------------+ <-- CpuMpData
1994 // +--------------------+ <-- CpuMpData->CpuData
1996 // +--------------------+ <-- CpuMpData->CpuInfoInHob
1997 // CPU_INFO_IN_HOB (N)
1998 // +--------------------+
2000 MonitorBuffer
= (UINT8
*) (Buffer
+ ApStackSize
* MaxLogicalProcessorNumber
);
2001 BackupBufferAddr
= (UINTN
) MonitorBuffer
+ MonitorFilterSize
* MaxLogicalProcessorNumber
;
2002 ApIdtBase
= ALIGN_VALUE (BackupBufferAddr
+ ApResetVectorSize
, 8);
2003 CpuMpData
= (CPU_MP_DATA
*) (ApIdtBase
+ VolatileRegisters
.Idtr
.Limit
+ 1);
2004 CpuMpData
->Buffer
= Buffer
;
2005 CpuMpData
->CpuApStackSize
= ApStackSize
;
2006 CpuMpData
->BackupBuffer
= BackupBufferAddr
;
2007 CpuMpData
->BackupBufferSize
= ApResetVectorSize
;
2008 CpuMpData
->WakeupBuffer
= (UINTN
) -1;
2009 CpuMpData
->CpuCount
= 1;
2010 CpuMpData
->BspNumber
= 0;
2011 CpuMpData
->WaitEvent
= NULL
;
2012 CpuMpData
->SwitchBspFlag
= FALSE
;
2013 CpuMpData
->CpuData
= (CPU_AP_DATA
*) (CpuMpData
+ 1);
2014 CpuMpData
->CpuInfoInHob
= (UINT64
) (UINTN
) (CpuMpData
->CpuData
+ MaxLogicalProcessorNumber
);
2015 InitializeSpinLock(&CpuMpData
->MpLock
);
2016 CpuMpData
->SevEsIsEnabled
= PcdGetBool (PcdSevEsIsEnabled
);
2017 CpuMpData
->SevEsAPBuffer
= (UINTN
) -1;
2018 CpuMpData
->GhcbBase
= PcdGet64 (PcdGhcbBase
);
2021 // Make sure no memory usage outside of the allocated buffer.
2023 ASSERT ((CpuMpData
->CpuInfoInHob
+ sizeof (CPU_INFO_IN_HOB
) * MaxLogicalProcessorNumber
) ==
2024 Buffer
+ BufferSize
);
2027 // Duplicate BSP's IDT to APs.
2028 // All APs share one separate IDT. So AP can get the address of CpuMpData by using IDTR.BASE + IDTR.LIMIT + 1
2030 CopyMem ((VOID
*)ApIdtBase
, (VOID
*)VolatileRegisters
.Idtr
.Base
, VolatileRegisters
.Idtr
.Limit
+ 1);
2031 VolatileRegisters
.Idtr
.Base
= ApIdtBase
;
2033 // Don't pass BSP's TR to APs to avoid AP init failure.
2035 VolatileRegisters
.Tr
= 0;
2036 CopyMem (&CpuMpData
->CpuData
[0].VolatileRegisters
, &VolatileRegisters
, sizeof (VolatileRegisters
));
2038 // Set BSP basic information
2040 InitializeApData (CpuMpData
, 0, 0, CpuMpData
->Buffer
+ ApStackSize
);
2042 // Save assembly code information
2044 CopyMem (&CpuMpData
->AddressMap
, &AddressMap
, sizeof (MP_ASSEMBLY_ADDRESS_MAP
));
2046 // Finally set AP loop mode
2048 CpuMpData
->ApLoopMode
= ApLoopMode
;
2049 DEBUG ((DEBUG_INFO
, "AP Loop Mode is %d\n", CpuMpData
->ApLoopMode
));
2051 CpuMpData
->WakeUpByInitSipiSipi
= (CpuMpData
->ApLoopMode
== ApInHltLoop
);
2054 // Set up APs wakeup signal buffer
2056 for (Index
= 0; Index
< MaxLogicalProcessorNumber
; Index
++) {
2057 CpuMpData
->CpuData
[Index
].StartupApSignal
=
2058 (UINT32
*)(MonitorBuffer
+ MonitorFilterSize
* Index
);
2061 // Enable the local APIC for Virtual Wire Mode.
2063 ProgramVirtualWireMode ();
2065 if (OldCpuMpData
== NULL
) {
2066 if (MaxLogicalProcessorNumber
> 1) {
2068 // Wakeup all APs and calculate the processor count in system
2070 CollectProcessorCount (CpuMpData
);
2074 // APs have been wakeup before, just get the CPU Information
2077 OldCpuMpData
->NewCpuMpData
= CpuMpData
;
2078 CpuMpData
->CpuCount
= OldCpuMpData
->CpuCount
;
2079 CpuMpData
->BspNumber
= OldCpuMpData
->BspNumber
;
2080 CpuMpData
->CpuInfoInHob
= OldCpuMpData
->CpuInfoInHob
;
2081 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
2082 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
2083 InitializeSpinLock(&CpuMpData
->CpuData
[Index
].ApLock
);
2084 CpuMpData
->CpuData
[Index
].CpuHealthy
= (CpuInfoInHob
[Index
].Health
== 0)? TRUE
:FALSE
;
2085 CpuMpData
->CpuData
[Index
].ApFunction
= 0;
2089 if (!GetMicrocodePatchInfoFromHob (
2090 &CpuMpData
->MicrocodePatchAddress
,
2091 &CpuMpData
->MicrocodePatchRegionSize
2094 // The microcode patch information cache HOB does not exist, which means
2095 // the microcode patches data has not been loaded into memory yet
2097 ShadowMicrocodeUpdatePatch (CpuMpData
);
2101 // Detect and apply Microcode on BSP
2103 MicrocodeDetect (CpuMpData
, CpuMpData
->BspNumber
);
2105 // Store BSP's MTRR setting
2107 MtrrGetAllMtrrs (&CpuMpData
->MtrrTable
);
2110 // Wakeup APs to do some AP initialize sync (Microcode & MTRR)
2112 if (CpuMpData
->CpuCount
> 1) {
2113 if (OldCpuMpData
!= NULL
) {
2115 // Only needs to use this flag for DXE phase to update the wake up
2116 // buffer. Wakeup buffer allocated in PEI phase is no longer valid
2119 CpuMpData
->InitFlag
= ApInitReconfig
;
2121 WakeUpAP (CpuMpData
, TRUE
, 0, ApInitializeSync
, CpuMpData
, TRUE
);
2123 // Wait for all APs finished initialization
2125 while (CpuMpData
->FinishedCount
< (CpuMpData
->CpuCount
- 1)) {
2128 if (OldCpuMpData
!= NULL
) {
2129 CpuMpData
->InitFlag
= ApInitDone
;
2131 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
2132 SetApState (&CpuMpData
->CpuData
[Index
], CpuStateIdle
);
2137 // Initialize global data for MP support
2139 InitMpGlobalData (CpuMpData
);
2145 Gets detailed MP-related information on the requested processor at the
2146 instant this call is made. This service may only be called from the BSP.
2148 @param[in] ProcessorNumber The handle number of processor.
2149 @param[out] ProcessorInfoBuffer A pointer to the buffer where information for
2150 the requested processor is deposited.
2151 @param[out] HealthData Return processor health data.
2153 @retval EFI_SUCCESS Processor information was returned.
2154 @retval EFI_DEVICE_ERROR The calling processor is an AP.
2155 @retval EFI_INVALID_PARAMETER ProcessorInfoBuffer is NULL.
2156 @retval EFI_NOT_FOUND The processor with the handle specified by
2157 ProcessorNumber does not exist in the platform.
2158 @retval EFI_NOT_READY MP Initialize Library is not initialized.
2163 MpInitLibGetProcessorInfo (
2164 IN UINTN ProcessorNumber
,
2165 OUT EFI_PROCESSOR_INFORMATION
*ProcessorInfoBuffer
,
2166 OUT EFI_HEALTH_FLAGS
*HealthData OPTIONAL
2169 CPU_MP_DATA
*CpuMpData
;
2171 CPU_INFO_IN_HOB
*CpuInfoInHob
;
2172 UINTN OriginalProcessorNumber
;
2174 CpuMpData
= GetCpuMpData ();
2175 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
2178 // Lower 24 bits contains the actual processor number.
2180 OriginalProcessorNumber
= ProcessorNumber
;
2181 ProcessorNumber
&= BIT24
- 1;
2184 // Check whether caller processor is BSP
2186 MpInitLibWhoAmI (&CallerNumber
);
2187 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2188 return EFI_DEVICE_ERROR
;
2191 if (ProcessorInfoBuffer
== NULL
) {
2192 return EFI_INVALID_PARAMETER
;
2195 if (ProcessorNumber
>= CpuMpData
->CpuCount
) {
2196 return EFI_NOT_FOUND
;
2199 ProcessorInfoBuffer
->ProcessorId
= (UINT64
) CpuInfoInHob
[ProcessorNumber
].ApicId
;
2200 ProcessorInfoBuffer
->StatusFlag
= 0;
2201 if (ProcessorNumber
== CpuMpData
->BspNumber
) {
2202 ProcessorInfoBuffer
->StatusFlag
|= PROCESSOR_AS_BSP_BIT
;
2204 if (CpuMpData
->CpuData
[ProcessorNumber
].CpuHealthy
) {
2205 ProcessorInfoBuffer
->StatusFlag
|= PROCESSOR_HEALTH_STATUS_BIT
;
2207 if (GetApState (&CpuMpData
->CpuData
[ProcessorNumber
]) == CpuStateDisabled
) {
2208 ProcessorInfoBuffer
->StatusFlag
&= ~PROCESSOR_ENABLED_BIT
;
2210 ProcessorInfoBuffer
->StatusFlag
|= PROCESSOR_ENABLED_BIT
;
2214 // Get processor location information
2216 GetProcessorLocationByApicId (
2217 CpuInfoInHob
[ProcessorNumber
].ApicId
,
2218 &ProcessorInfoBuffer
->Location
.Package
,
2219 &ProcessorInfoBuffer
->Location
.Core
,
2220 &ProcessorInfoBuffer
->Location
.Thread
2223 if ((OriginalProcessorNumber
& CPU_V2_EXTENDED_TOPOLOGY
) != 0) {
2224 GetProcessorLocation2ByApicId (
2225 CpuInfoInHob
[ProcessorNumber
].ApicId
,
2226 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Package
,
2227 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Die
,
2228 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Tile
,
2229 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Module
,
2230 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Core
,
2231 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Thread
2235 if (HealthData
!= NULL
) {
2236 HealthData
->Uint32
= CpuInfoInHob
[ProcessorNumber
].Health
;
2243 Worker function to switch the requested AP to be the BSP from that point onward.
2245 @param[in] ProcessorNumber The handle number of AP that is to become the new BSP.
2246 @param[in] EnableOldBSP If TRUE, then the old BSP will be listed as an
2247 enabled AP. Otherwise, it will be disabled.
2249 @retval EFI_SUCCESS BSP successfully switched.
2250 @retval others Failed to switch BSP.
2255 IN UINTN ProcessorNumber
,
2256 IN BOOLEAN EnableOldBSP
2259 CPU_MP_DATA
*CpuMpData
;
2262 MSR_IA32_APIC_BASE_REGISTER ApicBaseMsr
;
2263 BOOLEAN OldInterruptState
;
2264 BOOLEAN OldTimerInterruptState
;
2267 // Save and Disable Local APIC timer interrupt
2269 OldTimerInterruptState
= GetApicTimerInterruptState ();
2270 DisableApicTimerInterrupt ();
2272 // Before send both BSP and AP to a procedure to exchange their roles,
2273 // interrupt must be disabled. This is because during the exchange role
2274 // process, 2 CPU may use 1 stack. If interrupt happens, the stack will
2275 // be corrupted, since interrupt return address will be pushed to stack
2278 OldInterruptState
= SaveAndDisableInterrupts ();
2281 // Mask LINT0 & LINT1 for the old BSP
2283 DisableLvtInterrupts ();
2285 CpuMpData
= GetCpuMpData ();
2288 // Check whether caller processor is BSP
2290 MpInitLibWhoAmI (&CallerNumber
);
2291 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2292 return EFI_DEVICE_ERROR
;
2295 if (ProcessorNumber
>= CpuMpData
->CpuCount
) {
2296 return EFI_NOT_FOUND
;
2300 // Check whether specified AP is disabled
2302 State
= GetApState (&CpuMpData
->CpuData
[ProcessorNumber
]);
2303 if (State
== CpuStateDisabled
) {
2304 return EFI_INVALID_PARAMETER
;
2308 // Check whether ProcessorNumber specifies the current BSP
2310 if (ProcessorNumber
== CpuMpData
->BspNumber
) {
2311 return EFI_INVALID_PARAMETER
;
2315 // Check whether specified AP is busy
2317 if (State
== CpuStateBusy
) {
2318 return EFI_NOT_READY
;
2321 CpuMpData
->BSPInfo
.State
= CPU_SWITCH_STATE_IDLE
;
2322 CpuMpData
->APInfo
.State
= CPU_SWITCH_STATE_IDLE
;
2323 CpuMpData
->SwitchBspFlag
= TRUE
;
2324 CpuMpData
->NewBspNumber
= ProcessorNumber
;
2327 // Clear the BSP bit of MSR_IA32_APIC_BASE
2329 ApicBaseMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_APIC_BASE
);
2330 ApicBaseMsr
.Bits
.BSP
= 0;
2331 AsmWriteMsr64 (MSR_IA32_APIC_BASE
, ApicBaseMsr
.Uint64
);
2334 // Need to wakeUp AP (future BSP).
2336 WakeUpAP (CpuMpData
, FALSE
, ProcessorNumber
, FutureBSPProc
, CpuMpData
, TRUE
);
2338 AsmExchangeRole (&CpuMpData
->BSPInfo
, &CpuMpData
->APInfo
);
2341 // Set the BSP bit of MSR_IA32_APIC_BASE on new BSP
2343 ApicBaseMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_APIC_BASE
);
2344 ApicBaseMsr
.Bits
.BSP
= 1;
2345 AsmWriteMsr64 (MSR_IA32_APIC_BASE
, ApicBaseMsr
.Uint64
);
2346 ProgramVirtualWireMode ();
2349 // Wait for old BSP finished AP task
2351 while (GetApState (&CpuMpData
->CpuData
[CallerNumber
]) != CpuStateFinished
) {
2355 CpuMpData
->SwitchBspFlag
= FALSE
;
2357 // Set old BSP enable state
2359 if (!EnableOldBSP
) {
2360 SetApState (&CpuMpData
->CpuData
[CallerNumber
], CpuStateDisabled
);
2362 SetApState (&CpuMpData
->CpuData
[CallerNumber
], CpuStateIdle
);
2365 // Save new BSP number
2367 CpuMpData
->BspNumber
= (UINT32
) ProcessorNumber
;
2370 // Restore interrupt state.
2372 SetInterruptState (OldInterruptState
);
2374 if (OldTimerInterruptState
) {
2375 EnableApicTimerInterrupt ();
2382 Worker function to let the caller enable or disable an AP from this point onward.
2383 This service may only be called from the BSP.
2385 @param[in] ProcessorNumber The handle number of AP.
2386 @param[in] EnableAP Specifies the new state for the processor for
2387 enabled, FALSE for disabled.
2388 @param[in] HealthFlag If not NULL, a pointer to a value that specifies
2389 the new health status of the AP.
2391 @retval EFI_SUCCESS The specified AP was enabled or disabled successfully.
2392 @retval others Failed to Enable/Disable AP.
2396 EnableDisableApWorker (
2397 IN UINTN ProcessorNumber
,
2398 IN BOOLEAN EnableAP
,
2399 IN UINT32
*HealthFlag OPTIONAL
2402 CPU_MP_DATA
*CpuMpData
;
2405 CpuMpData
= GetCpuMpData ();
2408 // Check whether caller processor is BSP
2410 MpInitLibWhoAmI (&CallerNumber
);
2411 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2412 return EFI_DEVICE_ERROR
;
2415 if (ProcessorNumber
== CpuMpData
->BspNumber
) {
2416 return EFI_INVALID_PARAMETER
;
2419 if (ProcessorNumber
>= CpuMpData
->CpuCount
) {
2420 return EFI_NOT_FOUND
;
2424 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateDisabled
);
2426 ResetProcessorToIdleState (ProcessorNumber
);
2429 if (HealthFlag
!= NULL
) {
2430 CpuMpData
->CpuData
[ProcessorNumber
].CpuHealthy
=
2431 (BOOLEAN
) ((*HealthFlag
& PROCESSOR_HEALTH_STATUS_BIT
) != 0);
2438 This return the handle number for the calling processor. This service may be
2439 called from the BSP and APs.
2441 @param[out] ProcessorNumber Pointer to the handle number of AP.
2442 The range is from 0 to the total number of
2443 logical processors minus 1. The total number of
2444 logical processors can be retrieved by
2445 MpInitLibGetNumberOfProcessors().
2447 @retval EFI_SUCCESS The current processor handle number was returned
2449 @retval EFI_INVALID_PARAMETER ProcessorNumber is NULL.
2450 @retval EFI_NOT_READY MP Initialize Library is not initialized.
2456 OUT UINTN
*ProcessorNumber
2459 CPU_MP_DATA
*CpuMpData
;
2461 if (ProcessorNumber
== NULL
) {
2462 return EFI_INVALID_PARAMETER
;
2465 CpuMpData
= GetCpuMpData ();
2467 return GetProcessorNumber (CpuMpData
, ProcessorNumber
);
2471 Retrieves the number of logical processor in the platform and the number of
2472 those logical processors that are enabled on this boot. This service may only
2473 be called from the BSP.
2475 @param[out] NumberOfProcessors Pointer to the total number of logical
2476 processors in the system, including the BSP
2478 @param[out] NumberOfEnabledProcessors Pointer to the number of enabled logical
2479 processors that exist in system, including
2482 @retval EFI_SUCCESS The number of logical processors and enabled
2483 logical processors was retrieved.
2484 @retval EFI_DEVICE_ERROR The calling processor is an AP.
2485 @retval EFI_INVALID_PARAMETER NumberOfProcessors is NULL and NumberOfEnabledProcessors
2487 @retval EFI_NOT_READY MP Initialize Library is not initialized.
2492 MpInitLibGetNumberOfProcessors (
2493 OUT UINTN
*NumberOfProcessors
, OPTIONAL
2494 OUT UINTN
*NumberOfEnabledProcessors OPTIONAL
2497 CPU_MP_DATA
*CpuMpData
;
2499 UINTN ProcessorNumber
;
2500 UINTN EnabledProcessorNumber
;
2503 CpuMpData
= GetCpuMpData ();
2505 if ((NumberOfProcessors
== NULL
) && (NumberOfEnabledProcessors
== NULL
)) {
2506 return EFI_INVALID_PARAMETER
;
2510 // Check whether caller processor is BSP
2512 MpInitLibWhoAmI (&CallerNumber
);
2513 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2514 return EFI_DEVICE_ERROR
;
2517 ProcessorNumber
= CpuMpData
->CpuCount
;
2518 EnabledProcessorNumber
= 0;
2519 for (Index
= 0; Index
< ProcessorNumber
; Index
++) {
2520 if (GetApState (&CpuMpData
->CpuData
[Index
]) != CpuStateDisabled
) {
2521 EnabledProcessorNumber
++;
2525 if (NumberOfProcessors
!= NULL
) {
2526 *NumberOfProcessors
= ProcessorNumber
;
2528 if (NumberOfEnabledProcessors
!= NULL
) {
2529 *NumberOfEnabledProcessors
= EnabledProcessorNumber
;
2537 Worker function to execute a caller provided function on all enabled APs.
2539 @param[in] Procedure A pointer to the function to be run on
2540 enabled APs of the system.
2541 @param[in] SingleThread If TRUE, then all the enabled APs execute
2542 the function specified by Procedure one by
2543 one, in ascending order of processor handle
2544 number. If FALSE, then all the enabled APs
2545 execute the function specified by Procedure
2547 @param[in] ExcludeBsp Whether let BSP also trig this task.
2548 @param[in] WaitEvent The event created by the caller with CreateEvent()
2550 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
2551 APs to return from Procedure, either for
2552 blocking or non-blocking mode.
2553 @param[in] ProcedureArgument The parameter passed into Procedure for
2555 @param[out] FailedCpuList If all APs finish successfully, then its
2556 content is set to NULL. If not all APs
2557 finish before timeout expires, then its
2558 content is set to address of the buffer
2559 holding handle numbers of the failed APs.
2561 @retval EFI_SUCCESS In blocking mode, all APs have finished before
2562 the timeout expired.
2563 @retval EFI_SUCCESS In non-blocking mode, function has been dispatched
2565 @retval others Failed to Startup all APs.
2569 StartupAllCPUsWorker (
2570 IN EFI_AP_PROCEDURE Procedure
,
2571 IN BOOLEAN SingleThread
,
2572 IN BOOLEAN ExcludeBsp
,
2573 IN EFI_EVENT WaitEvent OPTIONAL
,
2574 IN UINTN TimeoutInMicroseconds
,
2575 IN VOID
*ProcedureArgument OPTIONAL
,
2576 OUT UINTN
**FailedCpuList OPTIONAL
2580 CPU_MP_DATA
*CpuMpData
;
2581 UINTN ProcessorCount
;
2582 UINTN ProcessorNumber
;
2584 CPU_AP_DATA
*CpuData
;
2585 BOOLEAN HasEnabledAp
;
2588 CpuMpData
= GetCpuMpData ();
2590 if (FailedCpuList
!= NULL
) {
2591 *FailedCpuList
= NULL
;
2594 if (CpuMpData
->CpuCount
== 1 && ExcludeBsp
) {
2595 return EFI_NOT_STARTED
;
2598 if (Procedure
== NULL
) {
2599 return EFI_INVALID_PARAMETER
;
2603 // Check whether caller processor is BSP
2605 MpInitLibWhoAmI (&CallerNumber
);
2606 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2607 return EFI_DEVICE_ERROR
;
2613 CheckAndUpdateApsStatus ();
2615 ProcessorCount
= CpuMpData
->CpuCount
;
2616 HasEnabledAp
= FALSE
;
2618 // Check whether all enabled APs are idle.
2619 // If any enabled AP is not idle, return EFI_NOT_READY.
2621 for (ProcessorNumber
= 0; ProcessorNumber
< ProcessorCount
; ProcessorNumber
++) {
2622 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
2623 if (ProcessorNumber
!= CpuMpData
->BspNumber
) {
2624 ApState
= GetApState (CpuData
);
2625 if (ApState
!= CpuStateDisabled
) {
2626 HasEnabledAp
= TRUE
;
2627 if (ApState
!= CpuStateIdle
) {
2629 // If any enabled APs are busy, return EFI_NOT_READY.
2631 return EFI_NOT_READY
;
2637 if (!HasEnabledAp
&& ExcludeBsp
) {
2639 // If no enabled AP exists and not include Bsp to do the procedure, return EFI_NOT_STARTED.
2641 return EFI_NOT_STARTED
;
2644 CpuMpData
->RunningCount
= 0;
2645 for (ProcessorNumber
= 0; ProcessorNumber
< ProcessorCount
; ProcessorNumber
++) {
2646 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
2647 CpuData
->Waiting
= FALSE
;
2648 if (ProcessorNumber
!= CpuMpData
->BspNumber
) {
2649 if (CpuData
->State
== CpuStateIdle
) {
2651 // Mark this processor as responsible for current calling.
2653 CpuData
->Waiting
= TRUE
;
2654 CpuMpData
->RunningCount
++;
2659 CpuMpData
->Procedure
= Procedure
;
2660 CpuMpData
->ProcArguments
= ProcedureArgument
;
2661 CpuMpData
->SingleThread
= SingleThread
;
2662 CpuMpData
->FinishedCount
= 0;
2663 CpuMpData
->FailedCpuList
= FailedCpuList
;
2664 CpuMpData
->ExpectedTime
= CalculateTimeout (
2665 TimeoutInMicroseconds
,
2666 &CpuMpData
->CurrentTime
2668 CpuMpData
->TotalTime
= 0;
2669 CpuMpData
->WaitEvent
= WaitEvent
;
2671 if (!SingleThread
) {
2672 WakeUpAP (CpuMpData
, TRUE
, 0, Procedure
, ProcedureArgument
, FALSE
);
2674 for (ProcessorNumber
= 0; ProcessorNumber
< ProcessorCount
; ProcessorNumber
++) {
2675 if (ProcessorNumber
== CallerNumber
) {
2678 if (CpuMpData
->CpuData
[ProcessorNumber
].Waiting
) {
2679 WakeUpAP (CpuMpData
, FALSE
, ProcessorNumber
, Procedure
, ProcedureArgument
, TRUE
);
2689 Procedure (ProcedureArgument
);
2692 Status
= EFI_SUCCESS
;
2693 if (WaitEvent
== NULL
) {
2695 Status
= CheckAllAPs ();
2696 } while (Status
== EFI_NOT_READY
);
2703 Worker function to let the caller get one enabled AP to execute a caller-provided
2706 @param[in] Procedure A pointer to the function to be run on
2707 enabled APs of the system.
2708 @param[in] ProcessorNumber The handle number of the AP.
2709 @param[in] WaitEvent The event created by the caller with CreateEvent()
2711 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
2712 APs to return from Procedure, either for
2713 blocking or non-blocking mode.
2714 @param[in] ProcedureArgument The parameter passed into Procedure for
2716 @param[out] Finished If AP returns from Procedure before the
2717 timeout expires, its content is set to TRUE.
2718 Otherwise, the value is set to FALSE.
2720 @retval EFI_SUCCESS In blocking mode, specified AP finished before
2721 the timeout expires.
2722 @retval others Failed to Startup AP.
2726 StartupThisAPWorker (
2727 IN EFI_AP_PROCEDURE Procedure
,
2728 IN UINTN ProcessorNumber
,
2729 IN EFI_EVENT WaitEvent OPTIONAL
,
2730 IN UINTN TimeoutInMicroseconds
,
2731 IN VOID
*ProcedureArgument OPTIONAL
,
2732 OUT BOOLEAN
*Finished OPTIONAL
2736 CPU_MP_DATA
*CpuMpData
;
2737 CPU_AP_DATA
*CpuData
;
2740 CpuMpData
= GetCpuMpData ();
2742 if (Finished
!= NULL
) {
2747 // Check whether caller processor is BSP
2749 MpInitLibWhoAmI (&CallerNumber
);
2750 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2751 return EFI_DEVICE_ERROR
;
2755 // Check whether processor with the handle specified by ProcessorNumber exists
2757 if (ProcessorNumber
>= CpuMpData
->CpuCount
) {
2758 return EFI_NOT_FOUND
;
2762 // Check whether specified processor is BSP
2764 if (ProcessorNumber
== CpuMpData
->BspNumber
) {
2765 return EFI_INVALID_PARAMETER
;
2769 // Check parameter Procedure
2771 if (Procedure
== NULL
) {
2772 return EFI_INVALID_PARAMETER
;
2778 CheckAndUpdateApsStatus ();
2781 // Check whether specified AP is disabled
2783 if (GetApState (&CpuMpData
->CpuData
[ProcessorNumber
]) == CpuStateDisabled
) {
2784 return EFI_INVALID_PARAMETER
;
2788 // If WaitEvent is not NULL, execute in non-blocking mode.
2789 // BSP saves data for CheckAPsStatus(), and returns EFI_SUCCESS.
2790 // CheckAPsStatus() will check completion and timeout periodically.
2792 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
2793 CpuData
->WaitEvent
= WaitEvent
;
2794 CpuData
->Finished
= Finished
;
2795 CpuData
->ExpectedTime
= CalculateTimeout (TimeoutInMicroseconds
, &CpuData
->CurrentTime
);
2796 CpuData
->TotalTime
= 0;
2798 WakeUpAP (CpuMpData
, FALSE
, ProcessorNumber
, Procedure
, ProcedureArgument
, TRUE
);
2801 // If WaitEvent is NULL, execute in blocking mode.
2802 // BSP checks AP's state until it finishes or TimeoutInMicrosecsond expires.
2804 Status
= EFI_SUCCESS
;
2805 if (WaitEvent
== NULL
) {
2807 Status
= CheckThisAP (ProcessorNumber
);
2808 } while (Status
== EFI_NOT_READY
);
2815 Get pointer to CPU MP Data structure from GUIDed HOB.
2817 @return The pointer to CPU MP Data structure.
2820 GetCpuMpDataFromGuidedHob (
2824 EFI_HOB_GUID_TYPE
*GuidHob
;
2826 CPU_MP_DATA
*CpuMpData
;
2829 GuidHob
= GetFirstGuidHob (&mCpuInitMpLibHobGuid
);
2830 if (GuidHob
!= NULL
) {
2831 DataInHob
= GET_GUID_HOB_DATA (GuidHob
);
2832 CpuMpData
= (CPU_MP_DATA
*) (*(UINTN
*) DataInHob
);
2838 This service executes a caller provided function on all enabled CPUs.
2840 @param[in] Procedure A pointer to the function to be run on
2841 enabled APs of the system. See type
2843 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
2844 APs to return from Procedure, either for
2845 blocking or non-blocking mode. Zero means
2846 infinity. TimeoutInMicroseconds is ignored
2848 @param[in] ProcedureArgument The parameter passed into Procedure for
2851 @retval EFI_SUCCESS In blocking mode, all CPUs have finished before
2852 the timeout expired.
2853 @retval EFI_SUCCESS In non-blocking mode, function has been dispatched
2854 to all enabled CPUs.
2855 @retval EFI_DEVICE_ERROR Caller processor is AP.
2856 @retval EFI_NOT_READY Any enabled APs are busy.
2857 @retval EFI_NOT_READY MP Initialize Library is not initialized.
2858 @retval EFI_TIMEOUT In blocking mode, the timeout expired before
2859 all enabled APs have finished.
2860 @retval EFI_INVALID_PARAMETER Procedure is NULL.
2865 MpInitLibStartupAllCPUs (
2866 IN EFI_AP_PROCEDURE Procedure
,
2867 IN UINTN TimeoutInMicroseconds
,
2868 IN VOID
*ProcedureArgument OPTIONAL
2871 return StartupAllCPUsWorker (
2876 TimeoutInMicroseconds
,