2 CPU MP Initialize Library common functions.
4 Copyright (c) 2016 - 2020, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2020, AMD Inc. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
12 #include <Library/VmgExitLib.h>
13 #include <Register/Amd/Fam17Msr.h>
14 #include <Register/Amd/Ghcb.h>
16 EFI_GUID mCpuInitMpLibHobGuid
= CPU_INIT_MP_LIB_HOB_GUID
;
20 The function will check if BSP Execute Disable is enabled.
22 DxeIpl may have enabled Execute Disable for BSP, APs need to
23 get the status and sync up the settings.
24 If BSP's CR0.Paging is not set, BSP execute Disble feature is
27 @retval TRUE BSP Execute Disable is enabled.
28 @retval FALSE BSP Execute Disable is not enabled.
31 IsBspExecuteDisableEnabled (
36 CPUID_EXTENDED_CPU_SIG_EDX Edx
;
37 MSR_IA32_EFER_REGISTER EferMsr
;
42 Cr0
.UintN
= AsmReadCr0 ();
43 if (Cr0
.Bits
.PG
!= 0) {
45 // If CR0 Paging bit is set
47 AsmCpuid (CPUID_EXTENDED_FUNCTION
, &Eax
, NULL
, NULL
, NULL
);
48 if (Eax
>= CPUID_EXTENDED_CPU_SIG
) {
49 AsmCpuid (CPUID_EXTENDED_CPU_SIG
, NULL
, NULL
, NULL
, &Edx
.Uint32
);
52 // Bit 20: Execute Disable Bit available.
54 if (Edx
.Bits
.NX
!= 0) {
55 EferMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_EFER
);
58 // Bit 11: Execute Disable Bit enable.
60 if (EferMsr
.Bits
.NXE
!= 0) {
71 Worker function for SwitchBSP().
73 Worker function for SwitchBSP(), assigned to the AP which is intended
76 @param[in] Buffer Pointer to CPU MP Data
84 CPU_MP_DATA
*DataInHob
;
86 DataInHob
= (CPU_MP_DATA
*) Buffer
;
87 AsmExchangeRole (&DataInHob
->APInfo
, &DataInHob
->BSPInfo
);
91 Get the Application Processors state.
93 @param[in] CpuData The pointer to CPU_AP_DATA of specified AP
99 IN CPU_AP_DATA
*CpuData
102 return CpuData
->State
;
106 Set the Application Processors state.
108 @param[in] CpuData The pointer to CPU_AP_DATA of specified AP
109 @param[in] State The AP status
113 IN CPU_AP_DATA
*CpuData
,
117 AcquireSpinLock (&CpuData
->ApLock
);
118 CpuData
->State
= State
;
119 ReleaseSpinLock (&CpuData
->ApLock
);
123 Save BSP's local APIC timer setting.
125 @param[in] CpuMpData Pointer to CPU MP Data
128 SaveLocalApicTimerSetting (
129 IN CPU_MP_DATA
*CpuMpData
133 // Record the current local APIC timer setting of BSP
136 &CpuMpData
->DivideValue
,
137 &CpuMpData
->PeriodicMode
,
140 CpuMpData
->CurrentTimerCount
= GetApicTimerCurrentCount ();
141 CpuMpData
->TimerInterruptState
= GetApicTimerInterruptState ();
145 Sync local APIC timer setting from BSP to AP.
147 @param[in] CpuMpData Pointer to CPU MP Data
150 SyncLocalApicTimerSetting (
151 IN CPU_MP_DATA
*CpuMpData
155 // Sync local APIC timer setting from BSP to AP
157 InitializeApicTimer (
158 CpuMpData
->DivideValue
,
159 CpuMpData
->CurrentTimerCount
,
160 CpuMpData
->PeriodicMode
,
164 // Disable AP's local APIC timer interrupt
166 DisableApicTimerInterrupt ();
170 Save the volatile registers required to be restored following INIT IPI.
172 @param[out] VolatileRegisters Returns buffer saved the volatile resisters
175 SaveVolatileRegisters (
176 OUT CPU_VOLATILE_REGISTERS
*VolatileRegisters
179 CPUID_VERSION_INFO_EDX VersionInfoEdx
;
181 VolatileRegisters
->Cr0
= AsmReadCr0 ();
182 VolatileRegisters
->Cr3
= AsmReadCr3 ();
183 VolatileRegisters
->Cr4
= AsmReadCr4 ();
185 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, NULL
, &VersionInfoEdx
.Uint32
);
186 if (VersionInfoEdx
.Bits
.DE
!= 0) {
188 // If processor supports Debugging Extensions feature
189 // by CPUID.[EAX=01H]:EDX.BIT2
191 VolatileRegisters
->Dr0
= AsmReadDr0 ();
192 VolatileRegisters
->Dr1
= AsmReadDr1 ();
193 VolatileRegisters
->Dr2
= AsmReadDr2 ();
194 VolatileRegisters
->Dr3
= AsmReadDr3 ();
195 VolatileRegisters
->Dr6
= AsmReadDr6 ();
196 VolatileRegisters
->Dr7
= AsmReadDr7 ();
199 AsmReadGdtr (&VolatileRegisters
->Gdtr
);
200 AsmReadIdtr (&VolatileRegisters
->Idtr
);
201 VolatileRegisters
->Tr
= AsmReadTr ();
205 Restore the volatile registers following INIT IPI.
207 @param[in] VolatileRegisters Pointer to volatile resisters
208 @param[in] IsRestoreDr TRUE: Restore DRx if supported
209 FALSE: Do not restore DRx
212 RestoreVolatileRegisters (
213 IN CPU_VOLATILE_REGISTERS
*VolatileRegisters
,
214 IN BOOLEAN IsRestoreDr
217 CPUID_VERSION_INFO_EDX VersionInfoEdx
;
218 IA32_TSS_DESCRIPTOR
*Tss
;
220 AsmWriteCr3 (VolatileRegisters
->Cr3
);
221 AsmWriteCr4 (VolatileRegisters
->Cr4
);
222 AsmWriteCr0 (VolatileRegisters
->Cr0
);
225 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, NULL
, &VersionInfoEdx
.Uint32
);
226 if (VersionInfoEdx
.Bits
.DE
!= 0) {
228 // If processor supports Debugging Extensions feature
229 // by CPUID.[EAX=01H]:EDX.BIT2
231 AsmWriteDr0 (VolatileRegisters
->Dr0
);
232 AsmWriteDr1 (VolatileRegisters
->Dr1
);
233 AsmWriteDr2 (VolatileRegisters
->Dr2
);
234 AsmWriteDr3 (VolatileRegisters
->Dr3
);
235 AsmWriteDr6 (VolatileRegisters
->Dr6
);
236 AsmWriteDr7 (VolatileRegisters
->Dr7
);
240 AsmWriteGdtr (&VolatileRegisters
->Gdtr
);
241 AsmWriteIdtr (&VolatileRegisters
->Idtr
);
242 if (VolatileRegisters
->Tr
!= 0 &&
243 VolatileRegisters
->Tr
< VolatileRegisters
->Gdtr
.Limit
) {
244 Tss
= (IA32_TSS_DESCRIPTOR
*)(VolatileRegisters
->Gdtr
.Base
+
245 VolatileRegisters
->Tr
);
246 if (Tss
->Bits
.P
== 1) {
247 Tss
->Bits
.Type
&= 0xD; // 1101 - Clear busy bit just in case
248 AsmWriteTr (VolatileRegisters
->Tr
);
254 Detect whether Mwait-monitor feature is supported.
256 @retval TRUE Mwait-monitor feature is supported.
257 @retval FALSE Mwait-monitor feature is not supported.
264 CPUID_VERSION_INFO_ECX VersionInfoEcx
;
266 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, &VersionInfoEcx
.Uint32
, NULL
);
267 return (VersionInfoEcx
.Bits
.MONITOR
== 1) ? TRUE
: FALSE
;
273 @param[out] MonitorFilterSize Returns the largest monitor-line size in bytes.
275 @return The AP loop mode.
279 OUT UINT32
*MonitorFilterSize
283 CPUID_MONITOR_MWAIT_EBX MonitorMwaitEbx
;
285 ASSERT (MonitorFilterSize
!= NULL
);
287 ApLoopMode
= PcdGet8 (PcdCpuApLoopMode
);
288 ASSERT (ApLoopMode
>= ApInHltLoop
&& ApLoopMode
<= ApInRunLoop
);
289 if (ApLoopMode
== ApInMwaitLoop
) {
290 if (!IsMwaitSupport ()) {
292 // If processor does not support MONITOR/MWAIT feature,
293 // force AP in Hlt-loop mode
295 ApLoopMode
= ApInHltLoop
;
298 if (PcdGetBool (PcdSevEsIsEnabled
)) {
300 // For SEV-ES, force AP in Hlt-loop mode in order to use the GHCB
301 // protocol for starting APs
303 ApLoopMode
= ApInHltLoop
;
307 if (ApLoopMode
!= ApInMwaitLoop
) {
308 *MonitorFilterSize
= sizeof (UINT32
);
311 // CPUID.[EAX=05H]:EBX.BIT0-15: Largest monitor-line size in bytes
312 // CPUID.[EAX=05H].EDX: C-states supported using MWAIT
314 AsmCpuid (CPUID_MONITOR_MWAIT
, NULL
, &MonitorMwaitEbx
.Uint32
, NULL
, NULL
);
315 *MonitorFilterSize
= MonitorMwaitEbx
.Bits
.LargestMonitorLineSize
;
322 Sort the APIC ID of all processors.
324 This function sorts the APIC ID of all processors so that processor number is
325 assigned in the ascending order of APIC ID which eases MP debugging.
327 @param[in] CpuMpData Pointer to PEI CPU MP Data
331 IN CPU_MP_DATA
*CpuMpData
338 CPU_INFO_IN_HOB CpuInfo
;
340 CPU_INFO_IN_HOB
*CpuInfoInHob
;
341 volatile UINT32
*StartupApSignal
;
343 ApCount
= CpuMpData
->CpuCount
- 1;
344 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
346 for (Index1
= 0; Index1
< ApCount
; Index1
++) {
349 // Sort key is the hardware default APIC ID
351 ApicId
= CpuInfoInHob
[Index1
].ApicId
;
352 for (Index2
= Index1
+ 1; Index2
<= ApCount
; Index2
++) {
353 if (ApicId
> CpuInfoInHob
[Index2
].ApicId
) {
355 ApicId
= CpuInfoInHob
[Index2
].ApicId
;
358 if (Index3
!= Index1
) {
359 CopyMem (&CpuInfo
, &CpuInfoInHob
[Index3
], sizeof (CPU_INFO_IN_HOB
));
361 &CpuInfoInHob
[Index3
],
362 &CpuInfoInHob
[Index1
],
363 sizeof (CPU_INFO_IN_HOB
)
365 CopyMem (&CpuInfoInHob
[Index1
], &CpuInfo
, sizeof (CPU_INFO_IN_HOB
));
368 // Also exchange the StartupApSignal.
370 StartupApSignal
= CpuMpData
->CpuData
[Index3
].StartupApSignal
;
371 CpuMpData
->CpuData
[Index3
].StartupApSignal
=
372 CpuMpData
->CpuData
[Index1
].StartupApSignal
;
373 CpuMpData
->CpuData
[Index1
].StartupApSignal
= StartupApSignal
;
378 // Get the processor number for the BSP
380 ApicId
= GetInitialApicId ();
381 for (Index1
= 0; Index1
< CpuMpData
->CpuCount
; Index1
++) {
382 if (CpuInfoInHob
[Index1
].ApicId
== ApicId
) {
383 CpuMpData
->BspNumber
= (UINT32
) Index1
;
391 Enable x2APIC mode on APs.
393 @param[in, out] Buffer Pointer to private data buffer.
401 SetApicMode (LOCAL_APIC_MODE_X2APIC
);
407 @param[in, out] Buffer Pointer to private data buffer.
415 CPU_MP_DATA
*CpuMpData
;
416 UINTN ProcessorNumber
;
419 CpuMpData
= (CPU_MP_DATA
*) Buffer
;
420 Status
= GetProcessorNumber (CpuMpData
, &ProcessorNumber
);
421 ASSERT_EFI_ERROR (Status
);
423 // Load microcode on AP
425 MicrocodeDetect (CpuMpData
, ProcessorNumber
);
427 // Sync BSP's MTRR table to AP
429 MtrrSetAllMtrrs (&CpuMpData
->MtrrTable
);
433 Find the current Processor number by APIC ID.
435 @param[in] CpuMpData Pointer to PEI CPU MP Data
436 @param[out] ProcessorNumber Return the pocessor number found
438 @retval EFI_SUCCESS ProcessorNumber is found and returned.
439 @retval EFI_NOT_FOUND ProcessorNumber is not found.
443 IN CPU_MP_DATA
*CpuMpData
,
444 OUT UINTN
*ProcessorNumber
447 UINTN TotalProcessorNumber
;
449 CPU_INFO_IN_HOB
*CpuInfoInHob
;
450 UINT32 CurrentApicId
;
452 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
454 TotalProcessorNumber
= CpuMpData
->CpuCount
;
455 CurrentApicId
= GetApicId ();
456 for (Index
= 0; Index
< TotalProcessorNumber
; Index
++) {
457 if (CpuInfoInHob
[Index
].ApicId
== CurrentApicId
) {
458 *ProcessorNumber
= Index
;
463 return EFI_NOT_FOUND
;
467 This function will get CPU count in the system.
469 @param[in] CpuMpData Pointer to PEI CPU MP Data
471 @return CPU count detected
474 CollectProcessorCount (
475 IN CPU_MP_DATA
*CpuMpData
479 CPU_INFO_IN_HOB
*CpuInfoInHob
;
483 // Send 1st broadcast IPI to APs to wakeup APs
485 CpuMpData
->InitFlag
= ApInitConfig
;
486 WakeUpAP (CpuMpData
, TRUE
, 0, NULL
, NULL
, TRUE
);
487 CpuMpData
->InitFlag
= ApInitDone
;
488 ASSERT (CpuMpData
->CpuCount
<= PcdGet32 (PcdCpuMaxLogicalProcessorNumber
));
490 // Wait for all APs finished the initialization
492 while (CpuMpData
->FinishedCount
< (CpuMpData
->CpuCount
- 1)) {
498 // Enable x2APIC mode if
499 // 1. Number of CPU is greater than 255; or
500 // 2. There are any logical processors reporting an Initial APIC ID of 255 or greater.
503 if (CpuMpData
->CpuCount
> 255) {
505 // If there are more than 255 processor found, force to enable X2APIC
509 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
510 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
511 if (CpuInfoInHob
[Index
].InitialApicId
>= 0xFF) {
519 DEBUG ((DEBUG_INFO
, "Force x2APIC mode!\n"));
521 // Wakeup all APs to enable x2APIC mode
523 WakeUpAP (CpuMpData
, TRUE
, 0, ApFuncEnableX2Apic
, NULL
, TRUE
);
525 // Wait for all known APs finished
527 while (CpuMpData
->FinishedCount
< (CpuMpData
->CpuCount
- 1)) {
531 // Enable x2APIC on BSP
533 SetApicMode (LOCAL_APIC_MODE_X2APIC
);
535 // Set BSP/Aps state to IDLE
537 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
538 SetApState (&CpuMpData
->CpuData
[Index
], CpuStateIdle
);
541 DEBUG ((DEBUG_INFO
, "APIC MODE is %d\n", GetApicMode ()));
543 // Sort BSP/Aps by CPU APIC ID in ascending order
545 SortApicId (CpuMpData
);
547 DEBUG ((DEBUG_INFO
, "MpInitLib: Find %d processors in system.\n", CpuMpData
->CpuCount
));
549 return CpuMpData
->CpuCount
;
553 Initialize CPU AP Data when AP is wakeup at the first time.
555 @param[in, out] CpuMpData Pointer to PEI CPU MP Data
556 @param[in] ProcessorNumber The handle number of processor
557 @param[in] BistData Processor BIST data
558 @param[in] ApTopOfStack Top of AP stack
563 IN OUT CPU_MP_DATA
*CpuMpData
,
564 IN UINTN ProcessorNumber
,
566 IN UINT64 ApTopOfStack
569 CPU_INFO_IN_HOB
*CpuInfoInHob
;
570 MSR_IA32_PLATFORM_ID_REGISTER PlatformIdMsr
;
572 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
573 CpuInfoInHob
[ProcessorNumber
].InitialApicId
= GetInitialApicId ();
574 CpuInfoInHob
[ProcessorNumber
].ApicId
= GetApicId ();
575 CpuInfoInHob
[ProcessorNumber
].Health
= BistData
;
576 CpuInfoInHob
[ProcessorNumber
].ApTopOfStack
= ApTopOfStack
;
578 CpuMpData
->CpuData
[ProcessorNumber
].Waiting
= FALSE
;
579 CpuMpData
->CpuData
[ProcessorNumber
].CpuHealthy
= (BistData
== 0) ? TRUE
: FALSE
;
582 // NOTE: PlatformId is not relevant on AMD platforms.
584 if (!StandardSignatureIsAuthenticAMD ()) {
585 PlatformIdMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_PLATFORM_ID
);
586 CpuMpData
->CpuData
[ProcessorNumber
].PlatformId
= (UINT8
)PlatformIdMsr
.Bits
.PlatformId
;
591 &CpuMpData
->CpuData
[ProcessorNumber
].ProcessorSignature
,
597 InitializeSpinLock(&CpuMpData
->CpuData
[ProcessorNumber
].ApLock
);
598 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateIdle
);
602 Get Protected mode code segment with 16-bit default addressing
603 from current GDT table.
605 @return Protected mode 16-bit code segment value.
609 GetProtectedMode16CS (
613 IA32_DESCRIPTOR GdtrDesc
;
614 IA32_SEGMENT_DESCRIPTOR
*GdtEntry
;
619 AsmReadGdtr (&GdtrDesc
);
620 GdtEntryCount
= (GdtrDesc
.Limit
+ 1) / sizeof (IA32_SEGMENT_DESCRIPTOR
);
621 GdtEntry
= (IA32_SEGMENT_DESCRIPTOR
*) GdtrDesc
.Base
;
622 for (Index
= 0; Index
< GdtEntryCount
; Index
++) {
623 if (GdtEntry
->Bits
.L
== 0 &&
624 GdtEntry
->Bits
.DB
== 0 &&
625 GdtEntry
->Bits
.Type
> 8) {
630 ASSERT (Index
!= GdtEntryCount
);
635 Get Protected mode code segment with 32-bit default addressing
636 from current GDT table.
638 @return Protected mode 32-bit code segment value.
642 GetProtectedMode32CS (
646 IA32_DESCRIPTOR GdtrDesc
;
647 IA32_SEGMENT_DESCRIPTOR
*GdtEntry
;
652 AsmReadGdtr (&GdtrDesc
);
653 GdtEntryCount
= (GdtrDesc
.Limit
+ 1) / sizeof (IA32_SEGMENT_DESCRIPTOR
);
654 GdtEntry
= (IA32_SEGMENT_DESCRIPTOR
*) GdtrDesc
.Base
;
655 for (Index
= 0; Index
< GdtEntryCount
; Index
++) {
656 if (GdtEntry
->Bits
.L
== 0 &&
657 GdtEntry
->Bits
.DB
== 1 &&
658 GdtEntry
->Bits
.Type
> 8) {
663 ASSERT (Index
!= GdtEntryCount
);
668 Reset an AP when in SEV-ES mode.
670 If successful, this function never returns.
672 @param[in] Ghcb Pointer to the GHCB
673 @param[in] CpuMpData Pointer to CPU MP Data
678 MpInitLibSevEsAPReset (
680 IN CPU_MP_DATA
*CpuMpData
683 UINT16 Code16
, Code32
;
688 Code16
= GetProtectedMode16CS ();
689 Code32
= GetProtectedMode32CS ();
691 if (CpuMpData
->WakeupBufferHigh
!= 0) {
692 APResetFn
= (AP_RESET
*) (CpuMpData
->WakeupBufferHigh
+ CpuMpData
->AddressMap
.SwitchToRealNoNxOffset
);
694 APResetFn
= (AP_RESET
*) (CpuMpData
->MpCpuExchangeInfo
->BufferStart
+ CpuMpData
->AddressMap
.SwitchToRealOffset
);
697 BufferStart
= CpuMpData
->MpCpuExchangeInfo
->BufferStart
;
698 StackStart
= CpuMpData
->SevEsAPResetStackStart
-
699 (AP_RESET_STACK_SIZE
* GetApicId ());
702 // This call never returns.
704 APResetFn (BufferStart
, Code16
, Code32
, StackStart
);
708 This function will be called from AP reset code if BSP uses WakeUpAP.
710 @param[in] ExchangeInfo Pointer to the MP exchange info buffer
711 @param[in] ApIndex Number of current executing AP
716 IN MP_CPU_EXCHANGE_INFO
*ExchangeInfo
,
720 CPU_MP_DATA
*CpuMpData
;
721 UINTN ProcessorNumber
;
722 EFI_AP_PROCEDURE Procedure
;
725 volatile UINT32
*ApStartupSignalBuffer
;
726 CPU_INFO_IN_HOB
*CpuInfoInHob
;
728 UINTN CurrentApicMode
;
731 // AP finished assembly code and begin to execute C code
733 CpuMpData
= ExchangeInfo
->CpuMpData
;
736 // AP's local APIC settings will be lost after received INIT IPI
737 // We need to re-initialize them at here
739 ProgramVirtualWireMode ();
741 // Mask the LINT0 and LINT1 so that AP doesn't enter the system timer interrupt handler.
743 DisableLvtInterrupts ();
744 SyncLocalApicTimerSetting (CpuMpData
);
746 CurrentApicMode
= GetApicMode ();
748 if (CpuMpData
->InitFlag
== ApInitConfig
) {
752 InterlockedIncrement ((UINT32
*) &CpuMpData
->CpuCount
);
753 ProcessorNumber
= ApIndex
;
755 // This is first time AP wakeup, get BIST information from AP stack
757 ApTopOfStack
= CpuMpData
->Buffer
+ (ProcessorNumber
+ 1) * CpuMpData
->CpuApStackSize
;
758 BistData
= *(UINT32
*) ((UINTN
) ApTopOfStack
- sizeof (UINTN
));
760 // CpuMpData->CpuData[0].VolatileRegisters is initialized based on BSP environment,
761 // to initialize AP in InitConfig path.
762 // NOTE: IDTR.BASE stored in CpuMpData->CpuData[0].VolatileRegisters points to a different IDT shared by all APs.
764 RestoreVolatileRegisters (&CpuMpData
->CpuData
[0].VolatileRegisters
, FALSE
);
765 InitializeApData (CpuMpData
, ProcessorNumber
, BistData
, ApTopOfStack
);
766 ApStartupSignalBuffer
= CpuMpData
->CpuData
[ProcessorNumber
].StartupApSignal
;
769 // Delay decrementing the APs executing count when SEV-ES is enabled
770 // to allow the APs to issue an AP_RESET_HOLD before the BSP possibly
771 // performs another INIT-SIPI-SIPI sequence.
773 if (!CpuMpData
->SevEsIsEnabled
) {
774 InterlockedDecrement ((UINT32
*) &CpuMpData
->MpCpuExchangeInfo
->NumApsExecuting
);
778 // Execute AP function if AP is ready
780 GetProcessorNumber (CpuMpData
, &ProcessorNumber
);
782 // Clear AP start-up signal when AP waken up
784 ApStartupSignalBuffer
= CpuMpData
->CpuData
[ProcessorNumber
].StartupApSignal
;
785 InterlockedCompareExchange32 (
786 (UINT32
*) ApStartupSignalBuffer
,
791 if (CpuMpData
->InitFlag
== ApInitReconfig
) {
793 // ApInitReconfig happens when:
794 // 1. AP is re-enabled after it's disabled, in either PEI or DXE phase.
795 // 2. AP is initialized in DXE phase.
796 // In either case, use the volatile registers value derived from BSP.
797 // NOTE: IDTR.BASE stored in CpuMpData->CpuData[0].VolatileRegisters points to a
798 // different IDT shared by all APs.
800 RestoreVolatileRegisters (&CpuMpData
->CpuData
[0].VolatileRegisters
, FALSE
);
802 if (CpuMpData
->ApLoopMode
== ApInHltLoop
) {
804 // Restore AP's volatile registers saved before AP is halted
806 RestoreVolatileRegisters (&CpuMpData
->CpuData
[ProcessorNumber
].VolatileRegisters
, TRUE
);
809 // The CPU driver might not flush TLB for APs on spot after updating
810 // page attributes. AP in mwait loop mode needs to take care of it when
817 if (GetApState (&CpuMpData
->CpuData
[ProcessorNumber
]) == CpuStateReady
) {
818 Procedure
= (EFI_AP_PROCEDURE
)CpuMpData
->CpuData
[ProcessorNumber
].ApFunction
;
819 Parameter
= (VOID
*) CpuMpData
->CpuData
[ProcessorNumber
].ApFunctionArgument
;
820 if (Procedure
!= NULL
) {
821 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateBusy
);
823 // Enable source debugging on AP function
827 // Invoke AP function here
829 Procedure (Parameter
);
830 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
831 if (CpuMpData
->SwitchBspFlag
) {
833 // Re-get the processor number due to BSP/AP maybe exchange in AP function
835 GetProcessorNumber (CpuMpData
, &ProcessorNumber
);
836 CpuMpData
->CpuData
[ProcessorNumber
].ApFunction
= 0;
837 CpuMpData
->CpuData
[ProcessorNumber
].ApFunctionArgument
= 0;
838 ApStartupSignalBuffer
= CpuMpData
->CpuData
[ProcessorNumber
].StartupApSignal
;
839 CpuInfoInHob
[ProcessorNumber
].ApTopOfStack
= CpuInfoInHob
[CpuMpData
->NewBspNumber
].ApTopOfStack
;
841 if (CpuInfoInHob
[ProcessorNumber
].ApicId
!= GetApicId () ||
842 CpuInfoInHob
[ProcessorNumber
].InitialApicId
!= GetInitialApicId ()) {
843 if (CurrentApicMode
!= GetApicMode ()) {
845 // If APIC mode change happened during AP function execution,
846 // we do not support APIC ID value changed.
852 // Re-get the CPU APICID and Initial APICID if they are changed
854 CpuInfoInHob
[ProcessorNumber
].ApicId
= GetApicId ();
855 CpuInfoInHob
[ProcessorNumber
].InitialApicId
= GetInitialApicId ();
860 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateFinished
);
865 // AP finished executing C code
867 InterlockedIncrement ((UINT32
*) &CpuMpData
->FinishedCount
);
870 // Place AP is specified loop mode
872 if (CpuMpData
->ApLoopMode
== ApInHltLoop
) {
874 // Save AP volatile registers
876 SaveVolatileRegisters (&CpuMpData
->CpuData
[ProcessorNumber
].VolatileRegisters
);
878 // Place AP in HLT-loop
881 DisableInterrupts ();
882 if (CpuMpData
->SevEsIsEnabled
) {
883 MSR_SEV_ES_GHCB_REGISTER Msr
;
887 BOOLEAN InterruptState
;
889 DoDecrement
= (BOOLEAN
) (CpuMpData
->InitFlag
== ApInitConfig
);
892 Msr
.GhcbPhysicalAddress
= AsmReadMsr64 (MSR_SEV_ES_GHCB
);
895 VmgInit (Ghcb
, &InterruptState
);
901 // Perform the delayed decrement just before issuing the first
902 // VMGEXIT with AP_RESET_HOLD.
904 InterlockedDecrement ((UINT32
*) &CpuMpData
->MpCpuExchangeInfo
->NumApsExecuting
);
907 Status
= VmgExit (Ghcb
, SVM_EXIT_AP_RESET_HOLD
, 0, 0);
908 if ((Status
== 0) && (Ghcb
->SaveArea
.SwExitInfo2
!= 0)) {
909 VmgDone (Ghcb
, InterruptState
);
913 VmgDone (Ghcb
, InterruptState
);
917 // Awakened in a new phase? Use the new CpuMpData
919 if (CpuMpData
->NewCpuMpData
!= NULL
) {
920 CpuMpData
= CpuMpData
->NewCpuMpData
;
923 MpInitLibSevEsAPReset (Ghcb
, CpuMpData
);
931 DisableInterrupts ();
932 if (CpuMpData
->ApLoopMode
== ApInMwaitLoop
) {
934 // Place AP in MWAIT-loop
936 AsmMonitor ((UINTN
) ApStartupSignalBuffer
, 0, 0);
937 if (*ApStartupSignalBuffer
!= WAKEUP_AP_SIGNAL
) {
939 // Check AP start-up signal again.
940 // If AP start-up signal is not set, place AP into
941 // the specified C-state
943 AsmMwait (CpuMpData
->ApTargetCState
<< 4, 0);
945 } else if (CpuMpData
->ApLoopMode
== ApInRunLoop
) {
947 // Place AP in Run-loop
955 // If AP start-up signal is written, AP is waken up
956 // otherwise place AP in loop again
958 if (*ApStartupSignalBuffer
== WAKEUP_AP_SIGNAL
) {
966 Wait for AP wakeup and write AP start-up signal till AP is waken up.
968 @param[in] ApStartupSignalBuffer Pointer to AP wakeup signal
972 IN
volatile UINT32
*ApStartupSignalBuffer
976 // If AP is waken up, StartupApSignal should be cleared.
977 // Otherwise, write StartupApSignal again till AP waken up.
979 while (InterlockedCompareExchange32 (
980 (UINT32
*) ApStartupSignalBuffer
,
989 This function will fill the exchange info structure.
991 @param[in] CpuMpData Pointer to CPU MP Data
995 FillExchangeInfoData (
996 IN CPU_MP_DATA
*CpuMpData
999 volatile MP_CPU_EXCHANGE_INFO
*ExchangeInfo
;
1001 IA32_SEGMENT_DESCRIPTOR
*Selector
;
1004 ExchangeInfo
= CpuMpData
->MpCpuExchangeInfo
;
1005 ExchangeInfo
->Lock
= 0;
1006 ExchangeInfo
->StackStart
= CpuMpData
->Buffer
;
1007 ExchangeInfo
->StackSize
= CpuMpData
->CpuApStackSize
;
1008 ExchangeInfo
->BufferStart
= CpuMpData
->WakeupBuffer
;
1009 ExchangeInfo
->ModeOffset
= CpuMpData
->AddressMap
.ModeEntryOffset
;
1011 ExchangeInfo
->CodeSegment
= AsmReadCs ();
1012 ExchangeInfo
->DataSegment
= AsmReadDs ();
1014 ExchangeInfo
->Cr3
= AsmReadCr3 ();
1016 ExchangeInfo
->CFunction
= (UINTN
) ApWakeupFunction
;
1017 ExchangeInfo
->ApIndex
= 0;
1018 ExchangeInfo
->NumApsExecuting
= 0;
1019 ExchangeInfo
->InitFlag
= (UINTN
) CpuMpData
->InitFlag
;
1020 ExchangeInfo
->CpuInfo
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
1021 ExchangeInfo
->CpuMpData
= CpuMpData
;
1023 ExchangeInfo
->EnableExecuteDisable
= IsBspExecuteDisableEnabled ();
1025 ExchangeInfo
->InitializeFloatingPointUnitsAddress
= (UINTN
)InitializeFloatingPointUnits
;
1028 // We can check either CPUID(7).ECX[bit16] or check CR4.LA57[bit12]
1029 // to determin whether 5-Level Paging is enabled.
1030 // CPUID(7).ECX[bit16] shows CPU's capability, CR4.LA57[bit12] shows
1031 // current system setting.
1032 // Using latter way is simpler because it also eliminates the needs to
1033 // check whether platform wants to enable it.
1035 Cr4
.UintN
= AsmReadCr4 ();
1036 ExchangeInfo
->Enable5LevelPaging
= (BOOLEAN
) (Cr4
.Bits
.LA57
== 1);
1037 DEBUG ((DEBUG_INFO
, "%a: 5-Level Paging = %d\n", gEfiCallerBaseName
, ExchangeInfo
->Enable5LevelPaging
));
1039 ExchangeInfo
->SevEsIsEnabled
= CpuMpData
->SevEsIsEnabled
;
1040 ExchangeInfo
->GhcbBase
= (UINTN
) CpuMpData
->GhcbBase
;
1043 // Get the BSP's data of GDT and IDT
1045 AsmReadGdtr ((IA32_DESCRIPTOR
*) &ExchangeInfo
->GdtrProfile
);
1046 AsmReadIdtr ((IA32_DESCRIPTOR
*) &ExchangeInfo
->IdtrProfile
);
1049 // Find a 32-bit code segment
1051 Selector
= (IA32_SEGMENT_DESCRIPTOR
*)ExchangeInfo
->GdtrProfile
.Base
;
1052 Size
= ExchangeInfo
->GdtrProfile
.Limit
+ 1;
1054 if (Selector
->Bits
.L
== 0 && Selector
->Bits
.Type
>= 8) {
1055 ExchangeInfo
->ModeTransitionSegment
=
1056 (UINT16
)((UINTN
)Selector
- ExchangeInfo
->GdtrProfile
.Base
);
1060 Size
-= sizeof (IA32_SEGMENT_DESCRIPTOR
);
1064 // Copy all 32-bit code and 64-bit code into memory with type of
1065 // EfiBootServicesCode to avoid page fault if NX memory protection is enabled.
1067 if (CpuMpData
->WakeupBufferHigh
!= 0) {
1068 Size
= CpuMpData
->AddressMap
.RendezvousFunnelSize
+
1069 CpuMpData
->AddressMap
.SwitchToRealSize
-
1070 CpuMpData
->AddressMap
.ModeTransitionOffset
;
1072 (VOID
*)CpuMpData
->WakeupBufferHigh
,
1073 CpuMpData
->AddressMap
.RendezvousFunnelAddress
+
1074 CpuMpData
->AddressMap
.ModeTransitionOffset
,
1078 ExchangeInfo
->ModeTransitionMemory
= (UINT32
)CpuMpData
->WakeupBufferHigh
;
1080 ExchangeInfo
->ModeTransitionMemory
= (UINT32
)
1081 (ExchangeInfo
->BufferStart
+ CpuMpData
->AddressMap
.ModeTransitionOffset
);
1084 ExchangeInfo
->ModeHighMemory
= ExchangeInfo
->ModeTransitionMemory
+
1085 (UINT32
)ExchangeInfo
->ModeOffset
-
1086 (UINT32
)CpuMpData
->AddressMap
.ModeTransitionOffset
;
1087 ExchangeInfo
->ModeHighSegment
= (UINT16
)ExchangeInfo
->CodeSegment
;
1091 Helper function that waits until the finished AP count reaches the specified
1092 limit, or the specified timeout elapses (whichever comes first).
1094 @param[in] CpuMpData Pointer to CPU MP Data.
1095 @param[in] FinishedApLimit The number of finished APs to wait for.
1096 @param[in] TimeLimit The number of microseconds to wait for.
1099 TimedWaitForApFinish (
1100 IN CPU_MP_DATA
*CpuMpData
,
1101 IN UINT32 FinishedApLimit
,
1106 Get available system memory below 1MB by specified size.
1108 @param[in] CpuMpData The pointer to CPU MP Data structure.
1111 BackupAndPrepareWakeupBuffer(
1112 IN CPU_MP_DATA
*CpuMpData
1116 (VOID
*) CpuMpData
->BackupBuffer
,
1117 (VOID
*) CpuMpData
->WakeupBuffer
,
1118 CpuMpData
->BackupBufferSize
1121 (VOID
*) CpuMpData
->WakeupBuffer
,
1122 (VOID
*) CpuMpData
->AddressMap
.RendezvousFunnelAddress
,
1123 CpuMpData
->AddressMap
.RendezvousFunnelSize
+
1124 CpuMpData
->AddressMap
.SwitchToRealSize
1129 Restore wakeup buffer data.
1131 @param[in] CpuMpData The pointer to CPU MP Data structure.
1134 RestoreWakeupBuffer(
1135 IN CPU_MP_DATA
*CpuMpData
1139 (VOID
*) CpuMpData
->WakeupBuffer
,
1140 (VOID
*) CpuMpData
->BackupBuffer
,
1141 CpuMpData
->BackupBufferSize
1146 Calculate the size of the reset vector.
1148 @param[in] AddressMap The pointer to Address Map structure.
1150 @return Total amount of memory required for the AP reset area
1154 GetApResetVectorSize (
1155 IN MP_ASSEMBLY_ADDRESS_MAP
*AddressMap
1160 Size
= AddressMap
->RendezvousFunnelSize
+
1161 AddressMap
->SwitchToRealSize
+
1162 sizeof (MP_CPU_EXCHANGE_INFO
);
1165 // The AP reset stack is only used by SEV-ES guests. Do not add to the
1166 // allocation if SEV-ES is not enabled.
1168 if (PcdGetBool (PcdSevEsIsEnabled
)) {
1170 // Stack location is based on APIC ID, so use the total number of
1171 // processors for calculating the total stack area.
1173 Size
+= AP_RESET_STACK_SIZE
* PcdGet32 (PcdCpuMaxLogicalProcessorNumber
);
1175 Size
= ALIGN_VALUE (Size
, CPU_STACK_ALIGNMENT
);
1182 Allocate reset vector buffer.
1184 @param[in, out] CpuMpData The pointer to CPU MP Data structure.
1187 AllocateResetVector (
1188 IN OUT CPU_MP_DATA
*CpuMpData
1191 UINTN ApResetVectorSize
;
1193 if (CpuMpData
->WakeupBuffer
== (UINTN
) -1) {
1194 ApResetVectorSize
= GetApResetVectorSize (&CpuMpData
->AddressMap
);
1196 CpuMpData
->WakeupBuffer
= GetWakeupBuffer (ApResetVectorSize
);
1197 CpuMpData
->MpCpuExchangeInfo
= (MP_CPU_EXCHANGE_INFO
*) (UINTN
)
1198 (CpuMpData
->WakeupBuffer
+
1199 CpuMpData
->AddressMap
.RendezvousFunnelSize
+
1200 CpuMpData
->AddressMap
.SwitchToRealSize
);
1201 CpuMpData
->WakeupBufferHigh
= GetModeTransitionBuffer (
1202 CpuMpData
->AddressMap
.RendezvousFunnelSize
+
1203 CpuMpData
->AddressMap
.SwitchToRealSize
-
1204 CpuMpData
->AddressMap
.ModeTransitionOffset
1207 // The reset stack starts at the end of the buffer.
1209 CpuMpData
->SevEsAPResetStackStart
= CpuMpData
->WakeupBuffer
+ ApResetVectorSize
;
1211 BackupAndPrepareWakeupBuffer (CpuMpData
);
1215 Free AP reset vector buffer.
1217 @param[in] CpuMpData The pointer to CPU MP Data structure.
1221 IN CPU_MP_DATA
*CpuMpData
1225 // If SEV-ES is enabled, the reset area is needed for AP parking and
1226 // and AP startup in the OS, so the reset area is reserved. Do not
1227 // perform the restore as this will overwrite memory which has data
1228 // needed by SEV-ES.
1230 if (!CpuMpData
->SevEsIsEnabled
) {
1231 RestoreWakeupBuffer (CpuMpData
);
1236 Allocate the SEV-ES AP jump table buffer.
1238 @param[in, out] CpuMpData The pointer to CPU MP Data structure.
1241 AllocateSevEsAPMemory (
1242 IN OUT CPU_MP_DATA
*CpuMpData
1245 if (CpuMpData
->SevEsAPBuffer
== (UINTN
) -1) {
1246 CpuMpData
->SevEsAPBuffer
=
1247 CpuMpData
->SevEsIsEnabled
? GetSevEsAPMemory () : 0;
1252 Program the SEV-ES AP jump table buffer.
1254 @param[in] SipiVector The SIPI vector used for the AP Reset
1261 SEV_ES_AP_JMP_FAR
*JmpFar
;
1262 UINT32 Offset
, InsnByte
;
1265 JmpFar
= (SEV_ES_AP_JMP_FAR
*) FixedPcdGet32 (PcdSevEsWorkAreaBase
);
1266 ASSERT (JmpFar
!= NULL
);
1269 // Obtain the address of the Segment/Rip location in the workarea.
1270 // This will be set to a value derived from the SIPI vector and will
1271 // be the memory address used for the far jump below.
1273 Offset
= FixedPcdGet32 (PcdSevEsWorkAreaBase
);
1274 Offset
+= sizeof (JmpFar
->InsnBuffer
);
1275 LoNib
= (UINT8
) Offset
;
1276 HiNib
= (UINT8
) (Offset
>> 8);
1279 // Program the workarea (which is the initial AP boot address) with
1280 // far jump to the SIPI vector (where XX and YY represent the
1281 // address of where the SIPI vector is stored.
1283 // JMP FAR [CS:XXYY] => 2E FF 2E YY XX
1286 JmpFar
->InsnBuffer
[InsnByte
++] = 0x2E; // CS override prefix
1287 JmpFar
->InsnBuffer
[InsnByte
++] = 0xFF; // JMP (FAR)
1288 JmpFar
->InsnBuffer
[InsnByte
++] = 0x2E; // ModRM (JMP memory location)
1289 JmpFar
->InsnBuffer
[InsnByte
++] = LoNib
; // YY offset ...
1290 JmpFar
->InsnBuffer
[InsnByte
++] = HiNib
; // XX offset ...
1293 // Program the Segment/Rip based on the SIPI vector (always at least
1294 // 16-byte aligned, so Rip is set to 0).
1297 JmpFar
->Segment
= (UINT16
) (SipiVector
>> 4);
1301 This function will be called by BSP to wakeup AP.
1303 @param[in] CpuMpData Pointer to CPU MP Data
1304 @param[in] Broadcast TRUE: Send broadcast IPI to all APs
1305 FALSE: Send IPI to AP by ApicId
1306 @param[in] ProcessorNumber The handle number of specified processor
1307 @param[in] Procedure The function to be invoked by AP
1308 @param[in] ProcedureArgument The argument to be passed into AP function
1309 @param[in] WakeUpDisabledAps Whether need to wake up disabled APs in broadcast mode.
1313 IN CPU_MP_DATA
*CpuMpData
,
1314 IN BOOLEAN Broadcast
,
1315 IN UINTN ProcessorNumber
,
1316 IN EFI_AP_PROCEDURE Procedure
, OPTIONAL
1317 IN VOID
*ProcedureArgument
, OPTIONAL
1318 IN BOOLEAN WakeUpDisabledAps
1321 volatile MP_CPU_EXCHANGE_INFO
*ExchangeInfo
;
1323 CPU_AP_DATA
*CpuData
;
1324 BOOLEAN ResetVectorRequired
;
1325 CPU_INFO_IN_HOB
*CpuInfoInHob
;
1327 CpuMpData
->FinishedCount
= 0;
1328 ResetVectorRequired
= FALSE
;
1330 if (CpuMpData
->WakeUpByInitSipiSipi
||
1331 CpuMpData
->InitFlag
!= ApInitDone
) {
1332 ResetVectorRequired
= TRUE
;
1333 AllocateResetVector (CpuMpData
);
1334 AllocateSevEsAPMemory (CpuMpData
);
1335 FillExchangeInfoData (CpuMpData
);
1336 SaveLocalApicTimerSetting (CpuMpData
);
1339 if (CpuMpData
->ApLoopMode
== ApInMwaitLoop
) {
1341 // Get AP target C-state each time when waking up AP,
1342 // for it maybe updated by platform again
1344 CpuMpData
->ApTargetCState
= PcdGet8 (PcdCpuApTargetCstate
);
1347 ExchangeInfo
= CpuMpData
->MpCpuExchangeInfo
;
1350 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
1351 if (Index
!= CpuMpData
->BspNumber
) {
1352 CpuData
= &CpuMpData
->CpuData
[Index
];
1354 // All AP(include disabled AP) will be woke up by INIT-SIPI-SIPI, but
1355 // the AP procedure will be skipped for disabled AP because AP state
1356 // is not CpuStateReady.
1358 if (GetApState (CpuData
) == CpuStateDisabled
&& !WakeUpDisabledAps
) {
1362 CpuData
->ApFunction
= (UINTN
) Procedure
;
1363 CpuData
->ApFunctionArgument
= (UINTN
) ProcedureArgument
;
1364 SetApState (CpuData
, CpuStateReady
);
1365 if (CpuMpData
->InitFlag
!= ApInitConfig
) {
1366 *(UINT32
*) CpuData
->StartupApSignal
= WAKEUP_AP_SIGNAL
;
1370 if (ResetVectorRequired
) {
1372 // For SEV-ES, the initial AP boot address will be defined by
1373 // PcdSevEsWorkAreaBase. The Segment/Rip must be the jump address
1374 // from the original INIT-SIPI-SIPI.
1376 if (CpuMpData
->SevEsIsEnabled
) {
1377 SetSevEsJumpTable (ExchangeInfo
->BufferStart
);
1383 SendInitSipiSipiAllExcludingSelf ((UINT32
) ExchangeInfo
->BufferStart
);
1385 if (CpuMpData
->InitFlag
== ApInitConfig
) {
1386 if (PcdGet32 (PcdCpuBootLogicalProcessorNumber
) > 0) {
1388 // The AP enumeration algorithm below is suitable only when the
1389 // platform can tell us the *exact* boot CPU count in advance.
1391 // The wait below finishes only when the detected AP count reaches
1392 // (PcdCpuBootLogicalProcessorNumber - 1), regardless of how long that
1393 // takes. If at least one AP fails to check in (meaning a platform
1394 // hardware bug), the detection hangs forever, by design. If the actual
1395 // boot CPU count in the system is higher than
1396 // PcdCpuBootLogicalProcessorNumber (meaning a platform
1397 // misconfiguration), then some APs may complete initialization after
1398 // the wait finishes, and cause undefined behavior.
1400 TimedWaitForApFinish (
1402 PcdGet32 (PcdCpuBootLogicalProcessorNumber
) - 1,
1403 MAX_UINT32
// approx. 71 minutes
1407 // The AP enumeration algorithm below is suitable for two use cases.
1409 // (1) The check-in time for an individual AP is bounded, and APs run
1410 // through their initialization routines strongly concurrently. In
1411 // particular, the number of concurrently running APs
1412 // ("NumApsExecuting") is never expected to fall to zero
1413 // *temporarily* -- it is expected to fall to zero only when all
1414 // APs have checked-in.
1416 // In this case, the platform is supposed to set
1417 // PcdCpuApInitTimeOutInMicroSeconds to a low-ish value (just long
1418 // enough for one AP to start initialization). The timeout will be
1419 // reached soon, and remaining APs are collected by watching
1420 // NumApsExecuting fall to zero. If NumApsExecuting falls to zero
1421 // mid-process, while some APs have not completed initialization,
1422 // the behavior is undefined.
1424 // (2) The check-in time for an individual AP is unbounded, and/or APs
1425 // may complete their initializations widely spread out. In
1426 // particular, some APs may finish initialization before some APs
1429 // In this case, the platform is supposed to set
1430 // PcdCpuApInitTimeOutInMicroSeconds to a high-ish value. The AP
1431 // enumeration will always take that long (except when the boot CPU
1432 // count happens to be maximal, that is,
1433 // PcdCpuMaxLogicalProcessorNumber). All APs are expected to
1434 // check-in before the timeout, and NumApsExecuting is assumed zero
1435 // at timeout. APs that miss the time-out may cause undefined
1438 TimedWaitForApFinish (
1440 PcdGet32 (PcdCpuMaxLogicalProcessorNumber
) - 1,
1441 PcdGet32 (PcdCpuApInitTimeOutInMicroSeconds
)
1444 while (CpuMpData
->MpCpuExchangeInfo
->NumApsExecuting
!= 0) {
1450 // Wait all APs waken up if this is not the 1st broadcast of SIPI
1452 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
1453 CpuData
= &CpuMpData
->CpuData
[Index
];
1454 if (Index
!= CpuMpData
->BspNumber
) {
1455 WaitApWakeup (CpuData
->StartupApSignal
);
1460 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
1461 CpuData
->ApFunction
= (UINTN
) Procedure
;
1462 CpuData
->ApFunctionArgument
= (UINTN
) ProcedureArgument
;
1463 SetApState (CpuData
, CpuStateReady
);
1465 // Wakeup specified AP
1467 ASSERT (CpuMpData
->InitFlag
!= ApInitConfig
);
1468 *(UINT32
*) CpuData
->StartupApSignal
= WAKEUP_AP_SIGNAL
;
1469 if (ResetVectorRequired
) {
1470 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
1473 // For SEV-ES, the initial AP boot address will be defined by
1474 // PcdSevEsWorkAreaBase. The Segment/Rip must be the jump address
1475 // from the original INIT-SIPI-SIPI.
1477 if (CpuMpData
->SevEsIsEnabled
) {
1478 SetSevEsJumpTable (ExchangeInfo
->BufferStart
);
1482 CpuInfoInHob
[ProcessorNumber
].ApicId
,
1483 (UINT32
) ExchangeInfo
->BufferStart
1487 // Wait specified AP waken up
1489 WaitApWakeup (CpuData
->StartupApSignal
);
1492 if (ResetVectorRequired
) {
1493 FreeResetVector (CpuMpData
);
1497 // After one round of Wakeup Ap actions, need to re-sync ApLoopMode with
1498 // WakeUpByInitSipiSipi flag. WakeUpByInitSipiSipi flag maybe changed by
1499 // S3SmmInitDone Ppi.
1501 CpuMpData
->WakeUpByInitSipiSipi
= (CpuMpData
->ApLoopMode
== ApInHltLoop
);
1505 Calculate timeout value and return the current performance counter value.
1507 Calculate the number of performance counter ticks required for a timeout.
1508 If TimeoutInMicroseconds is 0, return value is also 0, which is recognized
1511 @param[in] TimeoutInMicroseconds Timeout value in microseconds.
1512 @param[out] CurrentTime Returns the current value of the performance counter.
1514 @return Expected time stamp counter for timeout.
1515 If TimeoutInMicroseconds is 0, return value is also 0, which is recognized
1521 IN UINTN TimeoutInMicroseconds
,
1522 OUT UINT64
*CurrentTime
1525 UINT64 TimeoutInSeconds
;
1526 UINT64 TimestampCounterFreq
;
1529 // Read the current value of the performance counter
1531 *CurrentTime
= GetPerformanceCounter ();
1534 // If TimeoutInMicroseconds is 0, return value is also 0, which is recognized
1537 if (TimeoutInMicroseconds
== 0) {
1542 // GetPerformanceCounterProperties () returns the timestamp counter's frequency
1545 TimestampCounterFreq
= GetPerformanceCounterProperties (NULL
, NULL
);
1548 // Check the potential overflow before calculate the number of ticks for the timeout value.
1550 if (DivU64x64Remainder (MAX_UINT64
, TimeoutInMicroseconds
, NULL
) < TimestampCounterFreq
) {
1552 // Convert microseconds into seconds if direct multiplication overflows
1554 TimeoutInSeconds
= DivU64x32 (TimeoutInMicroseconds
, 1000000);
1556 // Assertion if the final tick count exceeds MAX_UINT64
1558 ASSERT (DivU64x64Remainder (MAX_UINT64
, TimeoutInSeconds
, NULL
) >= TimestampCounterFreq
);
1559 return MultU64x64 (TimestampCounterFreq
, TimeoutInSeconds
);
1562 // No overflow case, multiply the return value with TimeoutInMicroseconds and then divide
1563 // it by 1,000,000, to get the number of ticks for the timeout value.
1567 TimestampCounterFreq
,
1568 TimeoutInMicroseconds
1576 Checks whether timeout expires.
1578 Check whether the number of elapsed performance counter ticks required for
1579 a timeout condition has been reached.
1580 If Timeout is zero, which means infinity, return value is always FALSE.
1582 @param[in, out] PreviousTime On input, the value of the performance counter
1583 when it was last read.
1584 On output, the current value of the performance
1586 @param[in] TotalTime The total amount of elapsed time in performance
1588 @param[in] Timeout The number of performance counter ticks required
1589 to reach a timeout condition.
1591 @retval TRUE A timeout condition has been reached.
1592 @retval FALSE A timeout condition has not been reached.
1597 IN OUT UINT64
*PreviousTime
,
1598 IN UINT64
*TotalTime
,
1611 GetPerformanceCounterProperties (&Start
, &End
);
1612 Cycle
= End
- Start
;
1617 CurrentTime
= GetPerformanceCounter();
1618 Delta
= (INT64
) (CurrentTime
- *PreviousTime
);
1625 *TotalTime
+= Delta
;
1626 *PreviousTime
= CurrentTime
;
1627 if (*TotalTime
> Timeout
) {
1634 Helper function that waits until the finished AP count reaches the specified
1635 limit, or the specified timeout elapses (whichever comes first).
1637 @param[in] CpuMpData Pointer to CPU MP Data.
1638 @param[in] FinishedApLimit The number of finished APs to wait for.
1639 @param[in] TimeLimit The number of microseconds to wait for.
1642 TimedWaitForApFinish (
1643 IN CPU_MP_DATA
*CpuMpData
,
1644 IN UINT32 FinishedApLimit
,
1649 // CalculateTimeout() and CheckTimeout() consider a TimeLimit of 0
1650 // "infinity", so check for (TimeLimit == 0) explicitly.
1652 if (TimeLimit
== 0) {
1656 CpuMpData
->TotalTime
= 0;
1657 CpuMpData
->ExpectedTime
= CalculateTimeout (
1659 &CpuMpData
->CurrentTime
1661 while (CpuMpData
->FinishedCount
< FinishedApLimit
&&
1663 &CpuMpData
->CurrentTime
,
1664 &CpuMpData
->TotalTime
,
1665 CpuMpData
->ExpectedTime
1670 if (CpuMpData
->FinishedCount
>= FinishedApLimit
) {
1673 "%a: reached FinishedApLimit=%u in %Lu microseconds\n",
1676 DivU64x64Remainder (
1677 MultU64x32 (CpuMpData
->TotalTime
, 1000000),
1678 GetPerformanceCounterProperties (NULL
, NULL
),
1686 Reset an AP to Idle state.
1688 Any task being executed by the AP will be aborted and the AP
1689 will be waiting for a new task in Wait-For-SIPI state.
1691 @param[in] ProcessorNumber The handle number of processor.
1694 ResetProcessorToIdleState (
1695 IN UINTN ProcessorNumber
1698 CPU_MP_DATA
*CpuMpData
;
1700 CpuMpData
= GetCpuMpData ();
1702 CpuMpData
->InitFlag
= ApInitReconfig
;
1703 WakeUpAP (CpuMpData
, FALSE
, ProcessorNumber
, NULL
, NULL
, TRUE
);
1704 while (CpuMpData
->FinishedCount
< 1) {
1707 CpuMpData
->InitFlag
= ApInitDone
;
1709 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateIdle
);
1713 Searches for the next waiting AP.
1715 Search for the next AP that is put in waiting state by single-threaded StartupAllAPs().
1717 @param[out] NextProcessorNumber Pointer to the processor number of the next waiting AP.
1719 @retval EFI_SUCCESS The next waiting AP has been found.
1720 @retval EFI_NOT_FOUND No waiting AP exists.
1724 GetNextWaitingProcessorNumber (
1725 OUT UINTN
*NextProcessorNumber
1728 UINTN ProcessorNumber
;
1729 CPU_MP_DATA
*CpuMpData
;
1731 CpuMpData
= GetCpuMpData ();
1733 for (ProcessorNumber
= 0; ProcessorNumber
< CpuMpData
->CpuCount
; ProcessorNumber
++) {
1734 if (CpuMpData
->CpuData
[ProcessorNumber
].Waiting
) {
1735 *NextProcessorNumber
= ProcessorNumber
;
1740 return EFI_NOT_FOUND
;
1743 /** Checks status of specified AP.
1745 This function checks whether the specified AP has finished the task assigned
1746 by StartupThisAP(), and whether timeout expires.
1748 @param[in] ProcessorNumber The handle number of processor.
1750 @retval EFI_SUCCESS Specified AP has finished task assigned by StartupThisAPs().
1751 @retval EFI_TIMEOUT The timeout expires.
1752 @retval EFI_NOT_READY Specified AP has not finished task and timeout has not expired.
1756 IN UINTN ProcessorNumber
1759 CPU_MP_DATA
*CpuMpData
;
1760 CPU_AP_DATA
*CpuData
;
1762 CpuMpData
= GetCpuMpData ();
1763 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
1766 // Check the CPU state of AP. If it is CpuStateIdle, then the AP has finished its task.
1767 // Only BSP and corresponding AP access this unit of CPU Data. This means the AP will not modify the
1768 // value of state after setting the it to CpuStateIdle, so BSP can safely make use of its value.
1771 // If the AP finishes for StartupThisAP(), return EFI_SUCCESS.
1773 if (GetApState(CpuData
) == CpuStateFinished
) {
1774 if (CpuData
->Finished
!= NULL
) {
1775 *(CpuData
->Finished
) = TRUE
;
1777 SetApState (CpuData
, CpuStateIdle
);
1781 // If timeout expires for StartupThisAP(), report timeout.
1783 if (CheckTimeout (&CpuData
->CurrentTime
, &CpuData
->TotalTime
, CpuData
->ExpectedTime
)) {
1784 if (CpuData
->Finished
!= NULL
) {
1785 *(CpuData
->Finished
) = FALSE
;
1788 // Reset failed AP to idle state
1790 ResetProcessorToIdleState (ProcessorNumber
);
1795 return EFI_NOT_READY
;
1799 Checks status of all APs.
1801 This function checks whether all APs have finished task assigned by StartupAllAPs(),
1802 and whether timeout expires.
1804 @retval EFI_SUCCESS All APs have finished task assigned by StartupAllAPs().
1805 @retval EFI_TIMEOUT The timeout expires.
1806 @retval EFI_NOT_READY APs have not finished task and timeout has not expired.
1813 UINTN ProcessorNumber
;
1814 UINTN NextProcessorNumber
;
1817 CPU_MP_DATA
*CpuMpData
;
1818 CPU_AP_DATA
*CpuData
;
1820 CpuMpData
= GetCpuMpData ();
1822 NextProcessorNumber
= 0;
1825 // Go through all APs that are responsible for the StartupAllAPs().
1827 for (ProcessorNumber
= 0; ProcessorNumber
< CpuMpData
->CpuCount
; ProcessorNumber
++) {
1828 if (!CpuMpData
->CpuData
[ProcessorNumber
].Waiting
) {
1832 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
1834 // Check the CPU state of AP. If it is CpuStateIdle, then the AP has finished its task.
1835 // Only BSP and corresponding AP access this unit of CPU Data. This means the AP will not modify the
1836 // value of state after setting the it to CpuStateIdle, so BSP can safely make use of its value.
1838 if (GetApState(CpuData
) == CpuStateFinished
) {
1839 CpuMpData
->RunningCount
--;
1840 CpuMpData
->CpuData
[ProcessorNumber
].Waiting
= FALSE
;
1841 SetApState(CpuData
, CpuStateIdle
);
1844 // If in Single Thread mode, then search for the next waiting AP for execution.
1846 if (CpuMpData
->SingleThread
) {
1847 Status
= GetNextWaitingProcessorNumber (&NextProcessorNumber
);
1849 if (!EFI_ERROR (Status
)) {
1853 (UINT32
) NextProcessorNumber
,
1854 CpuMpData
->Procedure
,
1855 CpuMpData
->ProcArguments
,
1864 // If all APs finish, return EFI_SUCCESS.
1866 if (CpuMpData
->RunningCount
== 0) {
1871 // If timeout expires, report timeout.
1874 &CpuMpData
->CurrentTime
,
1875 &CpuMpData
->TotalTime
,
1876 CpuMpData
->ExpectedTime
)
1879 // If FailedCpuList is not NULL, record all failed APs in it.
1881 if (CpuMpData
->FailedCpuList
!= NULL
) {
1882 *CpuMpData
->FailedCpuList
=
1883 AllocatePool ((CpuMpData
->RunningCount
+ 1) * sizeof (UINTN
));
1884 ASSERT (*CpuMpData
->FailedCpuList
!= NULL
);
1888 for (ProcessorNumber
= 0; ProcessorNumber
< CpuMpData
->CpuCount
; ProcessorNumber
++) {
1890 // Check whether this processor is responsible for StartupAllAPs().
1892 if (CpuMpData
->CpuData
[ProcessorNumber
].Waiting
) {
1894 // Reset failed APs to idle state
1896 ResetProcessorToIdleState (ProcessorNumber
);
1897 CpuMpData
->CpuData
[ProcessorNumber
].Waiting
= FALSE
;
1898 if (CpuMpData
->FailedCpuList
!= NULL
) {
1899 (*CpuMpData
->FailedCpuList
)[ListIndex
++] = ProcessorNumber
;
1903 if (CpuMpData
->FailedCpuList
!= NULL
) {
1904 (*CpuMpData
->FailedCpuList
)[ListIndex
] = END_OF_CPU_LIST
;
1908 return EFI_NOT_READY
;
1912 MP Initialize Library initialization.
1914 This service will allocate AP reset vector and wakeup all APs to do APs
1917 This service must be invoked before all other MP Initialize Library
1918 service are invoked.
1920 @retval EFI_SUCCESS MP initialization succeeds.
1921 @retval Others MP initialization fails.
1926 MpInitLibInitialize (
1930 CPU_MP_DATA
*OldCpuMpData
;
1931 CPU_INFO_IN_HOB
*CpuInfoInHob
;
1932 UINT32 MaxLogicalProcessorNumber
;
1934 MP_ASSEMBLY_ADDRESS_MAP AddressMap
;
1935 CPU_VOLATILE_REGISTERS VolatileRegisters
;
1937 UINT32 MonitorFilterSize
;
1940 CPU_MP_DATA
*CpuMpData
;
1942 UINT8
*MonitorBuffer
;
1944 UINTN ApResetVectorSize
;
1945 UINTN BackupBufferAddr
;
1948 OldCpuMpData
= GetCpuMpDataFromGuidedHob ();
1949 if (OldCpuMpData
== NULL
) {
1950 MaxLogicalProcessorNumber
= PcdGet32(PcdCpuMaxLogicalProcessorNumber
);
1952 MaxLogicalProcessorNumber
= OldCpuMpData
->CpuCount
;
1954 ASSERT (MaxLogicalProcessorNumber
!= 0);
1956 AsmGetAddressMap (&AddressMap
);
1957 ApResetVectorSize
= GetApResetVectorSize (&AddressMap
);
1958 ApStackSize
= PcdGet32(PcdCpuApStackSize
);
1959 ApLoopMode
= GetApLoopMode (&MonitorFilterSize
);
1962 // Save BSP's Control registers for APs.
1964 SaveVolatileRegisters (&VolatileRegisters
);
1966 BufferSize
= ApStackSize
* MaxLogicalProcessorNumber
;
1967 BufferSize
+= MonitorFilterSize
* MaxLogicalProcessorNumber
;
1968 BufferSize
+= ApResetVectorSize
;
1969 BufferSize
= ALIGN_VALUE (BufferSize
, 8);
1970 BufferSize
+= VolatileRegisters
.Idtr
.Limit
+ 1;
1971 BufferSize
+= sizeof (CPU_MP_DATA
);
1972 BufferSize
+= (sizeof (CPU_AP_DATA
) + sizeof (CPU_INFO_IN_HOB
))* MaxLogicalProcessorNumber
;
1973 MpBuffer
= AllocatePages (EFI_SIZE_TO_PAGES (BufferSize
));
1974 ASSERT (MpBuffer
!= NULL
);
1975 ZeroMem (MpBuffer
, BufferSize
);
1976 Buffer
= (UINTN
) MpBuffer
;
1979 // The layout of the Buffer is as below:
1981 // +--------------------+ <-- Buffer
1983 // +--------------------+ <-- MonitorBuffer
1984 // AP Monitor Filters (N)
1985 // +--------------------+ <-- BackupBufferAddr (CpuMpData->BackupBuffer)
1987 // +--------------------+
1989 // +--------------------+ <-- ApIdtBase (8-byte boundary)
1990 // AP IDT All APs share one separate IDT. So AP can get address of CPU_MP_DATA from IDT Base.
1991 // +--------------------+ <-- CpuMpData
1993 // +--------------------+ <-- CpuMpData->CpuData
1995 // +--------------------+ <-- CpuMpData->CpuInfoInHob
1996 // CPU_INFO_IN_HOB (N)
1997 // +--------------------+
1999 MonitorBuffer
= (UINT8
*) (Buffer
+ ApStackSize
* MaxLogicalProcessorNumber
);
2000 BackupBufferAddr
= (UINTN
) MonitorBuffer
+ MonitorFilterSize
* MaxLogicalProcessorNumber
;
2001 ApIdtBase
= ALIGN_VALUE (BackupBufferAddr
+ ApResetVectorSize
, 8);
2002 CpuMpData
= (CPU_MP_DATA
*) (ApIdtBase
+ VolatileRegisters
.Idtr
.Limit
+ 1);
2003 CpuMpData
->Buffer
= Buffer
;
2004 CpuMpData
->CpuApStackSize
= ApStackSize
;
2005 CpuMpData
->BackupBuffer
= BackupBufferAddr
;
2006 CpuMpData
->BackupBufferSize
= ApResetVectorSize
;
2007 CpuMpData
->WakeupBuffer
= (UINTN
) -1;
2008 CpuMpData
->CpuCount
= 1;
2009 CpuMpData
->BspNumber
= 0;
2010 CpuMpData
->WaitEvent
= NULL
;
2011 CpuMpData
->SwitchBspFlag
= FALSE
;
2012 CpuMpData
->CpuData
= (CPU_AP_DATA
*) (CpuMpData
+ 1);
2013 CpuMpData
->CpuInfoInHob
= (UINT64
) (UINTN
) (CpuMpData
->CpuData
+ MaxLogicalProcessorNumber
);
2014 InitializeSpinLock(&CpuMpData
->MpLock
);
2015 CpuMpData
->SevEsIsEnabled
= PcdGetBool (PcdSevEsIsEnabled
);
2016 CpuMpData
->SevEsAPBuffer
= (UINTN
) -1;
2017 CpuMpData
->GhcbBase
= PcdGet64 (PcdGhcbBase
);
2020 // Make sure no memory usage outside of the allocated buffer.
2022 ASSERT ((CpuMpData
->CpuInfoInHob
+ sizeof (CPU_INFO_IN_HOB
) * MaxLogicalProcessorNumber
) ==
2023 Buffer
+ BufferSize
);
2026 // Duplicate BSP's IDT to APs.
2027 // All APs share one separate IDT. So AP can get the address of CpuMpData by using IDTR.BASE + IDTR.LIMIT + 1
2029 CopyMem ((VOID
*)ApIdtBase
, (VOID
*)VolatileRegisters
.Idtr
.Base
, VolatileRegisters
.Idtr
.Limit
+ 1);
2030 VolatileRegisters
.Idtr
.Base
= ApIdtBase
;
2032 // Don't pass BSP's TR to APs to avoid AP init failure.
2034 VolatileRegisters
.Tr
= 0;
2035 CopyMem (&CpuMpData
->CpuData
[0].VolatileRegisters
, &VolatileRegisters
, sizeof (VolatileRegisters
));
2037 // Set BSP basic information
2039 InitializeApData (CpuMpData
, 0, 0, CpuMpData
->Buffer
+ ApStackSize
);
2041 // Save assembly code information
2043 CopyMem (&CpuMpData
->AddressMap
, &AddressMap
, sizeof (MP_ASSEMBLY_ADDRESS_MAP
));
2045 // Finally set AP loop mode
2047 CpuMpData
->ApLoopMode
= ApLoopMode
;
2048 DEBUG ((DEBUG_INFO
, "AP Loop Mode is %d\n", CpuMpData
->ApLoopMode
));
2050 CpuMpData
->WakeUpByInitSipiSipi
= (CpuMpData
->ApLoopMode
== ApInHltLoop
);
2053 // Set up APs wakeup signal buffer
2055 for (Index
= 0; Index
< MaxLogicalProcessorNumber
; Index
++) {
2056 CpuMpData
->CpuData
[Index
].StartupApSignal
=
2057 (UINT32
*)(MonitorBuffer
+ MonitorFilterSize
* Index
);
2060 // Enable the local APIC for Virtual Wire Mode.
2062 ProgramVirtualWireMode ();
2064 if (OldCpuMpData
== NULL
) {
2065 if (MaxLogicalProcessorNumber
> 1) {
2067 // Wakeup all APs and calculate the processor count in system
2069 CollectProcessorCount (CpuMpData
);
2073 // APs have been wakeup before, just get the CPU Information
2076 OldCpuMpData
->NewCpuMpData
= CpuMpData
;
2077 CpuMpData
->CpuCount
= OldCpuMpData
->CpuCount
;
2078 CpuMpData
->BspNumber
= OldCpuMpData
->BspNumber
;
2079 CpuMpData
->CpuInfoInHob
= OldCpuMpData
->CpuInfoInHob
;
2080 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
2081 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
2082 InitializeSpinLock(&CpuMpData
->CpuData
[Index
].ApLock
);
2083 CpuMpData
->CpuData
[Index
].CpuHealthy
= (CpuInfoInHob
[Index
].Health
== 0)? TRUE
:FALSE
;
2084 CpuMpData
->CpuData
[Index
].ApFunction
= 0;
2088 if (!GetMicrocodePatchInfoFromHob (
2089 &CpuMpData
->MicrocodePatchAddress
,
2090 &CpuMpData
->MicrocodePatchRegionSize
2093 // The microcode patch information cache HOB does not exist, which means
2094 // the microcode patches data has not been loaded into memory yet
2096 ShadowMicrocodeUpdatePatch (CpuMpData
);
2100 // Detect and apply Microcode on BSP
2102 MicrocodeDetect (CpuMpData
, CpuMpData
->BspNumber
);
2104 // Store BSP's MTRR setting
2106 MtrrGetAllMtrrs (&CpuMpData
->MtrrTable
);
2109 // Wakeup APs to do some AP initialize sync (Microcode & MTRR)
2111 if (CpuMpData
->CpuCount
> 1) {
2112 if (OldCpuMpData
!= NULL
) {
2114 // Only needs to use this flag for DXE phase to update the wake up
2115 // buffer. Wakeup buffer allocated in PEI phase is no longer valid
2118 CpuMpData
->InitFlag
= ApInitReconfig
;
2120 WakeUpAP (CpuMpData
, TRUE
, 0, ApInitializeSync
, CpuMpData
, TRUE
);
2122 // Wait for all APs finished initialization
2124 while (CpuMpData
->FinishedCount
< (CpuMpData
->CpuCount
- 1)) {
2127 if (OldCpuMpData
!= NULL
) {
2128 CpuMpData
->InitFlag
= ApInitDone
;
2130 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
2131 SetApState (&CpuMpData
->CpuData
[Index
], CpuStateIdle
);
2136 // Initialize global data for MP support
2138 InitMpGlobalData (CpuMpData
);
2144 Gets detailed MP-related information on the requested processor at the
2145 instant this call is made. This service may only be called from the BSP.
2147 @param[in] ProcessorNumber The handle number of processor.
2148 @param[out] ProcessorInfoBuffer A pointer to the buffer where information for
2149 the requested processor is deposited.
2150 @param[out] HealthData Return processor health data.
2152 @retval EFI_SUCCESS Processor information was returned.
2153 @retval EFI_DEVICE_ERROR The calling processor is an AP.
2154 @retval EFI_INVALID_PARAMETER ProcessorInfoBuffer is NULL.
2155 @retval EFI_NOT_FOUND The processor with the handle specified by
2156 ProcessorNumber does not exist in the platform.
2157 @retval EFI_NOT_READY MP Initialize Library is not initialized.
2162 MpInitLibGetProcessorInfo (
2163 IN UINTN ProcessorNumber
,
2164 OUT EFI_PROCESSOR_INFORMATION
*ProcessorInfoBuffer
,
2165 OUT EFI_HEALTH_FLAGS
*HealthData OPTIONAL
2168 CPU_MP_DATA
*CpuMpData
;
2170 CPU_INFO_IN_HOB
*CpuInfoInHob
;
2171 UINTN OriginalProcessorNumber
;
2173 CpuMpData
= GetCpuMpData ();
2174 CpuInfoInHob
= (CPU_INFO_IN_HOB
*) (UINTN
) CpuMpData
->CpuInfoInHob
;
2177 // Lower 24 bits contains the actual processor number.
2179 OriginalProcessorNumber
= ProcessorNumber
;
2180 ProcessorNumber
&= BIT24
- 1;
2183 // Check whether caller processor is BSP
2185 MpInitLibWhoAmI (&CallerNumber
);
2186 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2187 return EFI_DEVICE_ERROR
;
2190 if (ProcessorInfoBuffer
== NULL
) {
2191 return EFI_INVALID_PARAMETER
;
2194 if (ProcessorNumber
>= CpuMpData
->CpuCount
) {
2195 return EFI_NOT_FOUND
;
2198 ProcessorInfoBuffer
->ProcessorId
= (UINT64
) CpuInfoInHob
[ProcessorNumber
].ApicId
;
2199 ProcessorInfoBuffer
->StatusFlag
= 0;
2200 if (ProcessorNumber
== CpuMpData
->BspNumber
) {
2201 ProcessorInfoBuffer
->StatusFlag
|= PROCESSOR_AS_BSP_BIT
;
2203 if (CpuMpData
->CpuData
[ProcessorNumber
].CpuHealthy
) {
2204 ProcessorInfoBuffer
->StatusFlag
|= PROCESSOR_HEALTH_STATUS_BIT
;
2206 if (GetApState (&CpuMpData
->CpuData
[ProcessorNumber
]) == CpuStateDisabled
) {
2207 ProcessorInfoBuffer
->StatusFlag
&= ~PROCESSOR_ENABLED_BIT
;
2209 ProcessorInfoBuffer
->StatusFlag
|= PROCESSOR_ENABLED_BIT
;
2213 // Get processor location information
2215 GetProcessorLocationByApicId (
2216 CpuInfoInHob
[ProcessorNumber
].ApicId
,
2217 &ProcessorInfoBuffer
->Location
.Package
,
2218 &ProcessorInfoBuffer
->Location
.Core
,
2219 &ProcessorInfoBuffer
->Location
.Thread
2222 if ((OriginalProcessorNumber
& CPU_V2_EXTENDED_TOPOLOGY
) != 0) {
2223 GetProcessorLocation2ByApicId (
2224 CpuInfoInHob
[ProcessorNumber
].ApicId
,
2225 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Package
,
2226 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Die
,
2227 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Tile
,
2228 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Module
,
2229 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Core
,
2230 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Thread
2234 if (HealthData
!= NULL
) {
2235 HealthData
->Uint32
= CpuInfoInHob
[ProcessorNumber
].Health
;
2242 Worker function to switch the requested AP to be the BSP from that point onward.
2244 @param[in] ProcessorNumber The handle number of AP that is to become the new BSP.
2245 @param[in] EnableOldBSP If TRUE, then the old BSP will be listed as an
2246 enabled AP. Otherwise, it will be disabled.
2248 @retval EFI_SUCCESS BSP successfully switched.
2249 @retval others Failed to switch BSP.
2254 IN UINTN ProcessorNumber
,
2255 IN BOOLEAN EnableOldBSP
2258 CPU_MP_DATA
*CpuMpData
;
2261 MSR_IA32_APIC_BASE_REGISTER ApicBaseMsr
;
2262 BOOLEAN OldInterruptState
;
2263 BOOLEAN OldTimerInterruptState
;
2266 // Save and Disable Local APIC timer interrupt
2268 OldTimerInterruptState
= GetApicTimerInterruptState ();
2269 DisableApicTimerInterrupt ();
2271 // Before send both BSP and AP to a procedure to exchange their roles,
2272 // interrupt must be disabled. This is because during the exchange role
2273 // process, 2 CPU may use 1 stack. If interrupt happens, the stack will
2274 // be corrupted, since interrupt return address will be pushed to stack
2277 OldInterruptState
= SaveAndDisableInterrupts ();
2280 // Mask LINT0 & LINT1 for the old BSP
2282 DisableLvtInterrupts ();
2284 CpuMpData
= GetCpuMpData ();
2287 // Check whether caller processor is BSP
2289 MpInitLibWhoAmI (&CallerNumber
);
2290 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2291 return EFI_DEVICE_ERROR
;
2294 if (ProcessorNumber
>= CpuMpData
->CpuCount
) {
2295 return EFI_NOT_FOUND
;
2299 // Check whether specified AP is disabled
2301 State
= GetApState (&CpuMpData
->CpuData
[ProcessorNumber
]);
2302 if (State
== CpuStateDisabled
) {
2303 return EFI_INVALID_PARAMETER
;
2307 // Check whether ProcessorNumber specifies the current BSP
2309 if (ProcessorNumber
== CpuMpData
->BspNumber
) {
2310 return EFI_INVALID_PARAMETER
;
2314 // Check whether specified AP is busy
2316 if (State
== CpuStateBusy
) {
2317 return EFI_NOT_READY
;
2320 CpuMpData
->BSPInfo
.State
= CPU_SWITCH_STATE_IDLE
;
2321 CpuMpData
->APInfo
.State
= CPU_SWITCH_STATE_IDLE
;
2322 CpuMpData
->SwitchBspFlag
= TRUE
;
2323 CpuMpData
->NewBspNumber
= ProcessorNumber
;
2326 // Clear the BSP bit of MSR_IA32_APIC_BASE
2328 ApicBaseMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_APIC_BASE
);
2329 ApicBaseMsr
.Bits
.BSP
= 0;
2330 AsmWriteMsr64 (MSR_IA32_APIC_BASE
, ApicBaseMsr
.Uint64
);
2333 // Need to wakeUp AP (future BSP).
2335 WakeUpAP (CpuMpData
, FALSE
, ProcessorNumber
, FutureBSPProc
, CpuMpData
, TRUE
);
2337 AsmExchangeRole (&CpuMpData
->BSPInfo
, &CpuMpData
->APInfo
);
2340 // Set the BSP bit of MSR_IA32_APIC_BASE on new BSP
2342 ApicBaseMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_APIC_BASE
);
2343 ApicBaseMsr
.Bits
.BSP
= 1;
2344 AsmWriteMsr64 (MSR_IA32_APIC_BASE
, ApicBaseMsr
.Uint64
);
2345 ProgramVirtualWireMode ();
2348 // Wait for old BSP finished AP task
2350 while (GetApState (&CpuMpData
->CpuData
[CallerNumber
]) != CpuStateFinished
) {
2354 CpuMpData
->SwitchBspFlag
= FALSE
;
2356 // Set old BSP enable state
2358 if (!EnableOldBSP
) {
2359 SetApState (&CpuMpData
->CpuData
[CallerNumber
], CpuStateDisabled
);
2361 SetApState (&CpuMpData
->CpuData
[CallerNumber
], CpuStateIdle
);
2364 // Save new BSP number
2366 CpuMpData
->BspNumber
= (UINT32
) ProcessorNumber
;
2369 // Restore interrupt state.
2371 SetInterruptState (OldInterruptState
);
2373 if (OldTimerInterruptState
) {
2374 EnableApicTimerInterrupt ();
2381 Worker function to let the caller enable or disable an AP from this point onward.
2382 This service may only be called from the BSP.
2384 @param[in] ProcessorNumber The handle number of AP.
2385 @param[in] EnableAP Specifies the new state for the processor for
2386 enabled, FALSE for disabled.
2387 @param[in] HealthFlag If not NULL, a pointer to a value that specifies
2388 the new health status of the AP.
2390 @retval EFI_SUCCESS The specified AP was enabled or disabled successfully.
2391 @retval others Failed to Enable/Disable AP.
2395 EnableDisableApWorker (
2396 IN UINTN ProcessorNumber
,
2397 IN BOOLEAN EnableAP
,
2398 IN UINT32
*HealthFlag OPTIONAL
2401 CPU_MP_DATA
*CpuMpData
;
2404 CpuMpData
= GetCpuMpData ();
2407 // Check whether caller processor is BSP
2409 MpInitLibWhoAmI (&CallerNumber
);
2410 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2411 return EFI_DEVICE_ERROR
;
2414 if (ProcessorNumber
== CpuMpData
->BspNumber
) {
2415 return EFI_INVALID_PARAMETER
;
2418 if (ProcessorNumber
>= CpuMpData
->CpuCount
) {
2419 return EFI_NOT_FOUND
;
2423 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateDisabled
);
2425 ResetProcessorToIdleState (ProcessorNumber
);
2428 if (HealthFlag
!= NULL
) {
2429 CpuMpData
->CpuData
[ProcessorNumber
].CpuHealthy
=
2430 (BOOLEAN
) ((*HealthFlag
& PROCESSOR_HEALTH_STATUS_BIT
) != 0);
2437 This return the handle number for the calling processor. This service may be
2438 called from the BSP and APs.
2440 @param[out] ProcessorNumber Pointer to the handle number of AP.
2441 The range is from 0 to the total number of
2442 logical processors minus 1. The total number of
2443 logical processors can be retrieved by
2444 MpInitLibGetNumberOfProcessors().
2446 @retval EFI_SUCCESS The current processor handle number was returned
2448 @retval EFI_INVALID_PARAMETER ProcessorNumber is NULL.
2449 @retval EFI_NOT_READY MP Initialize Library is not initialized.
2455 OUT UINTN
*ProcessorNumber
2458 CPU_MP_DATA
*CpuMpData
;
2460 if (ProcessorNumber
== NULL
) {
2461 return EFI_INVALID_PARAMETER
;
2464 CpuMpData
= GetCpuMpData ();
2466 return GetProcessorNumber (CpuMpData
, ProcessorNumber
);
2470 Retrieves the number of logical processor in the platform and the number of
2471 those logical processors that are enabled on this boot. This service may only
2472 be called from the BSP.
2474 @param[out] NumberOfProcessors Pointer to the total number of logical
2475 processors in the system, including the BSP
2477 @param[out] NumberOfEnabledProcessors Pointer to the number of enabled logical
2478 processors that exist in system, including
2481 @retval EFI_SUCCESS The number of logical processors and enabled
2482 logical processors was retrieved.
2483 @retval EFI_DEVICE_ERROR The calling processor is an AP.
2484 @retval EFI_INVALID_PARAMETER NumberOfProcessors is NULL and NumberOfEnabledProcessors
2486 @retval EFI_NOT_READY MP Initialize Library is not initialized.
2491 MpInitLibGetNumberOfProcessors (
2492 OUT UINTN
*NumberOfProcessors
, OPTIONAL
2493 OUT UINTN
*NumberOfEnabledProcessors OPTIONAL
2496 CPU_MP_DATA
*CpuMpData
;
2498 UINTN ProcessorNumber
;
2499 UINTN EnabledProcessorNumber
;
2502 CpuMpData
= GetCpuMpData ();
2504 if ((NumberOfProcessors
== NULL
) && (NumberOfEnabledProcessors
== NULL
)) {
2505 return EFI_INVALID_PARAMETER
;
2509 // Check whether caller processor is BSP
2511 MpInitLibWhoAmI (&CallerNumber
);
2512 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2513 return EFI_DEVICE_ERROR
;
2516 ProcessorNumber
= CpuMpData
->CpuCount
;
2517 EnabledProcessorNumber
= 0;
2518 for (Index
= 0; Index
< ProcessorNumber
; Index
++) {
2519 if (GetApState (&CpuMpData
->CpuData
[Index
]) != CpuStateDisabled
) {
2520 EnabledProcessorNumber
++;
2524 if (NumberOfProcessors
!= NULL
) {
2525 *NumberOfProcessors
= ProcessorNumber
;
2527 if (NumberOfEnabledProcessors
!= NULL
) {
2528 *NumberOfEnabledProcessors
= EnabledProcessorNumber
;
2536 Worker function to execute a caller provided function on all enabled APs.
2538 @param[in] Procedure A pointer to the function to be run on
2539 enabled APs of the system.
2540 @param[in] SingleThread If TRUE, then all the enabled APs execute
2541 the function specified by Procedure one by
2542 one, in ascending order of processor handle
2543 number. If FALSE, then all the enabled APs
2544 execute the function specified by Procedure
2546 @param[in] ExcludeBsp Whether let BSP also trig this task.
2547 @param[in] WaitEvent The event created by the caller with CreateEvent()
2549 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
2550 APs to return from Procedure, either for
2551 blocking or non-blocking mode.
2552 @param[in] ProcedureArgument The parameter passed into Procedure for
2554 @param[out] FailedCpuList If all APs finish successfully, then its
2555 content is set to NULL. If not all APs
2556 finish before timeout expires, then its
2557 content is set to address of the buffer
2558 holding handle numbers of the failed APs.
2560 @retval EFI_SUCCESS In blocking mode, all APs have finished before
2561 the timeout expired.
2562 @retval EFI_SUCCESS In non-blocking mode, function has been dispatched
2564 @retval others Failed to Startup all APs.
2568 StartupAllCPUsWorker (
2569 IN EFI_AP_PROCEDURE Procedure
,
2570 IN BOOLEAN SingleThread
,
2571 IN BOOLEAN ExcludeBsp
,
2572 IN EFI_EVENT WaitEvent OPTIONAL
,
2573 IN UINTN TimeoutInMicroseconds
,
2574 IN VOID
*ProcedureArgument OPTIONAL
,
2575 OUT UINTN
**FailedCpuList OPTIONAL
2579 CPU_MP_DATA
*CpuMpData
;
2580 UINTN ProcessorCount
;
2581 UINTN ProcessorNumber
;
2583 CPU_AP_DATA
*CpuData
;
2584 BOOLEAN HasEnabledAp
;
2587 CpuMpData
= GetCpuMpData ();
2589 if (FailedCpuList
!= NULL
) {
2590 *FailedCpuList
= NULL
;
2593 if (CpuMpData
->CpuCount
== 1 && ExcludeBsp
) {
2594 return EFI_NOT_STARTED
;
2597 if (Procedure
== NULL
) {
2598 return EFI_INVALID_PARAMETER
;
2602 // Check whether caller processor is BSP
2604 MpInitLibWhoAmI (&CallerNumber
);
2605 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2606 return EFI_DEVICE_ERROR
;
2612 CheckAndUpdateApsStatus ();
2614 ProcessorCount
= CpuMpData
->CpuCount
;
2615 HasEnabledAp
= FALSE
;
2617 // Check whether all enabled APs are idle.
2618 // If any enabled AP is not idle, return EFI_NOT_READY.
2620 for (ProcessorNumber
= 0; ProcessorNumber
< ProcessorCount
; ProcessorNumber
++) {
2621 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
2622 if (ProcessorNumber
!= CpuMpData
->BspNumber
) {
2623 ApState
= GetApState (CpuData
);
2624 if (ApState
!= CpuStateDisabled
) {
2625 HasEnabledAp
= TRUE
;
2626 if (ApState
!= CpuStateIdle
) {
2628 // If any enabled APs are busy, return EFI_NOT_READY.
2630 return EFI_NOT_READY
;
2636 if (!HasEnabledAp
&& ExcludeBsp
) {
2638 // If no enabled AP exists and not include Bsp to do the procedure, return EFI_NOT_STARTED.
2640 return EFI_NOT_STARTED
;
2643 CpuMpData
->RunningCount
= 0;
2644 for (ProcessorNumber
= 0; ProcessorNumber
< ProcessorCount
; ProcessorNumber
++) {
2645 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
2646 CpuData
->Waiting
= FALSE
;
2647 if (ProcessorNumber
!= CpuMpData
->BspNumber
) {
2648 if (CpuData
->State
== CpuStateIdle
) {
2650 // Mark this processor as responsible for current calling.
2652 CpuData
->Waiting
= TRUE
;
2653 CpuMpData
->RunningCount
++;
2658 CpuMpData
->Procedure
= Procedure
;
2659 CpuMpData
->ProcArguments
= ProcedureArgument
;
2660 CpuMpData
->SingleThread
= SingleThread
;
2661 CpuMpData
->FinishedCount
= 0;
2662 CpuMpData
->FailedCpuList
= FailedCpuList
;
2663 CpuMpData
->ExpectedTime
= CalculateTimeout (
2664 TimeoutInMicroseconds
,
2665 &CpuMpData
->CurrentTime
2667 CpuMpData
->TotalTime
= 0;
2668 CpuMpData
->WaitEvent
= WaitEvent
;
2670 if (!SingleThread
) {
2671 WakeUpAP (CpuMpData
, TRUE
, 0, Procedure
, ProcedureArgument
, FALSE
);
2673 for (ProcessorNumber
= 0; ProcessorNumber
< ProcessorCount
; ProcessorNumber
++) {
2674 if (ProcessorNumber
== CallerNumber
) {
2677 if (CpuMpData
->CpuData
[ProcessorNumber
].Waiting
) {
2678 WakeUpAP (CpuMpData
, FALSE
, ProcessorNumber
, Procedure
, ProcedureArgument
, TRUE
);
2688 Procedure (ProcedureArgument
);
2691 Status
= EFI_SUCCESS
;
2692 if (WaitEvent
== NULL
) {
2694 Status
= CheckAllAPs ();
2695 } while (Status
== EFI_NOT_READY
);
2702 Worker function to let the caller get one enabled AP to execute a caller-provided
2705 @param[in] Procedure A pointer to the function to be run on
2706 enabled APs of the system.
2707 @param[in] ProcessorNumber The handle number of the AP.
2708 @param[in] WaitEvent The event created by the caller with CreateEvent()
2710 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
2711 APs to return from Procedure, either for
2712 blocking or non-blocking mode.
2713 @param[in] ProcedureArgument The parameter passed into Procedure for
2715 @param[out] Finished If AP returns from Procedure before the
2716 timeout expires, its content is set to TRUE.
2717 Otherwise, the value is set to FALSE.
2719 @retval EFI_SUCCESS In blocking mode, specified AP finished before
2720 the timeout expires.
2721 @retval others Failed to Startup AP.
2725 StartupThisAPWorker (
2726 IN EFI_AP_PROCEDURE Procedure
,
2727 IN UINTN ProcessorNumber
,
2728 IN EFI_EVENT WaitEvent OPTIONAL
,
2729 IN UINTN TimeoutInMicroseconds
,
2730 IN VOID
*ProcedureArgument OPTIONAL
,
2731 OUT BOOLEAN
*Finished OPTIONAL
2735 CPU_MP_DATA
*CpuMpData
;
2736 CPU_AP_DATA
*CpuData
;
2739 CpuMpData
= GetCpuMpData ();
2741 if (Finished
!= NULL
) {
2746 // Check whether caller processor is BSP
2748 MpInitLibWhoAmI (&CallerNumber
);
2749 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2750 return EFI_DEVICE_ERROR
;
2754 // Check whether processor with the handle specified by ProcessorNumber exists
2756 if (ProcessorNumber
>= CpuMpData
->CpuCount
) {
2757 return EFI_NOT_FOUND
;
2761 // Check whether specified processor is BSP
2763 if (ProcessorNumber
== CpuMpData
->BspNumber
) {
2764 return EFI_INVALID_PARAMETER
;
2768 // Check parameter Procedure
2770 if (Procedure
== NULL
) {
2771 return EFI_INVALID_PARAMETER
;
2777 CheckAndUpdateApsStatus ();
2780 // Check whether specified AP is disabled
2782 if (GetApState (&CpuMpData
->CpuData
[ProcessorNumber
]) == CpuStateDisabled
) {
2783 return EFI_INVALID_PARAMETER
;
2787 // If WaitEvent is not NULL, execute in non-blocking mode.
2788 // BSP saves data for CheckAPsStatus(), and returns EFI_SUCCESS.
2789 // CheckAPsStatus() will check completion and timeout periodically.
2791 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
2792 CpuData
->WaitEvent
= WaitEvent
;
2793 CpuData
->Finished
= Finished
;
2794 CpuData
->ExpectedTime
= CalculateTimeout (TimeoutInMicroseconds
, &CpuData
->CurrentTime
);
2795 CpuData
->TotalTime
= 0;
2797 WakeUpAP (CpuMpData
, FALSE
, ProcessorNumber
, Procedure
, ProcedureArgument
, TRUE
);
2800 // If WaitEvent is NULL, execute in blocking mode.
2801 // BSP checks AP's state until it finishes or TimeoutInMicrosecsond expires.
2803 Status
= EFI_SUCCESS
;
2804 if (WaitEvent
== NULL
) {
2806 Status
= CheckThisAP (ProcessorNumber
);
2807 } while (Status
== EFI_NOT_READY
);
2814 Get pointer to CPU MP Data structure from GUIDed HOB.
2816 @return The pointer to CPU MP Data structure.
2819 GetCpuMpDataFromGuidedHob (
2823 EFI_HOB_GUID_TYPE
*GuidHob
;
2825 CPU_MP_DATA
*CpuMpData
;
2828 GuidHob
= GetFirstGuidHob (&mCpuInitMpLibHobGuid
);
2829 if (GuidHob
!= NULL
) {
2830 DataInHob
= GET_GUID_HOB_DATA (GuidHob
);
2831 CpuMpData
= (CPU_MP_DATA
*) (*(UINTN
*) DataInHob
);
2837 This service executes a caller provided function on all enabled CPUs.
2839 @param[in] Procedure A pointer to the function to be run on
2840 enabled APs of the system. See type
2842 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
2843 APs to return from Procedure, either for
2844 blocking or non-blocking mode. Zero means
2845 infinity. TimeoutInMicroseconds is ignored
2847 @param[in] ProcedureArgument The parameter passed into Procedure for
2850 @retval EFI_SUCCESS In blocking mode, all CPUs have finished before
2851 the timeout expired.
2852 @retval EFI_SUCCESS In non-blocking mode, function has been dispatched
2853 to all enabled CPUs.
2854 @retval EFI_DEVICE_ERROR Caller processor is AP.
2855 @retval EFI_NOT_READY Any enabled APs are busy.
2856 @retval EFI_NOT_READY MP Initialize Library is not initialized.
2857 @retval EFI_TIMEOUT In blocking mode, the timeout expired before
2858 all enabled APs have finished.
2859 @retval EFI_INVALID_PARAMETER Procedure is NULL.
2864 MpInitLibStartupAllCPUs (
2865 IN EFI_AP_PROCEDURE Procedure
,
2866 IN UINTN TimeoutInMicroseconds
,
2867 IN VOID
*ProcedureArgument OPTIONAL
2870 return StartupAllCPUsWorker (
2875 TimeoutInMicroseconds
,